We're using the Route53 DNS module to satisfy ACME challenges but are hitting a strange bug. For some reason, the TXT record created is malformed and wrongly duplicates the main zone as part of the subdomain (causing an endless verification timeout loop).
We are running caddy in a container on an EC2 machine.
If we manually create the correct TXT record (via the Route53 web console) and use the value from the wrongly auto-created TXT record, verification passes. The wrongly created TXT record is then automatically cleaned up, and the manually created one stays present.
root@ip-10-0-0-17:~# docker logs caddy
{"level":"info","ts":1631101549.0512967,"msg":"using provided configuration","config_file":"/etc/caddy/Caddyfile","config_adapter":"caddyfile"}
{"level":"info","ts":1631101549.0558064,"logger":"admin","msg":"admin endpoint started","address":"tcp/localhost:2019","enforce_origin":false,"origins":["localhost:2019","[::1]:2019","127.0.0.1:2019"]}
{"level":"info","ts":1631101549.0587218,"logger":"http","msg":"server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS","server_name":"srv0","https_port":443}
{"level":"info","ts":1631101549.0589776,"logger":"http","msg":"enabling automatic HTTP->HTTPS redirects","server_name":"srv0"}
{"level":"info","ts":1631101549.06152,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0xc0002e12d0"}
{"level":"info","ts":1631101549.0621595,"logger":"http","msg":"enabling automatic TLS certificate management","domains":["www.mydomain.com"]}
{"level":"info","ts":1631101549.0632653,"msg":"autosaved config","file":"/config/caddy/autosave.json"}
{"level":"info","ts":1631101549.0637987,"msg":"serving initial configuration"}
{"level":"info","ts":1631101549.0639155,"logger":"tls","msg":"cleaned up storage units"}
{"level":"info","ts":1631101549.0643346,"logger":"tls.obtain","msg":"acquiring lock","identifier":"www.mydomain.com"}
{"level":"info","ts":1631101549.0663319,"logger":"tls.obtain","msg":"lock acquired","identifier":"www.mydomain.com"}
{"level":"info","ts":1631101549.981814,"logger":"tls.issuance.acme","msg":"waiting on internal rate limiter","identifiers":["www.mydomain.com"]}
{"level":"info","ts":1631101549.9818606,"logger":"tls.issuance.acme","msg":"done waiting on internal rate limiter","identifiers":["www.mydomain.com"]}
{"level":"info","ts":1631101550.898957,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"www.mydomain.com","challenge_type":"dns-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"error","ts":1631101551.5204628,"logger":"tls.issuance.acme.acme_client","msg":"cleaning up solver","identifier":"www.mydomain.com","challenge_type":"dns-01","error":"no memory of presenting a DNS record for www.mydomain.com (probably OK if presenting failed)"}
{"level":"error","ts":1631101551.7166426,"logger":"tls.obtain","msg":"will retry","error":"[www.mydomain.com] Obtain: [www.mydomain.com] solving challenges: presenting for challenge: adding temporary record for zone mydomain.com.: InvalidChangeBatch: InvalidChangeBatch: [Tried to create resource record set [name='_acme-challenge.www.mydomain.com.mydomain.com.', type='TXT'] but it already exists]\n\tstatus code: 400, request id: 4d3c4fea-681a-4c09-8965-7fd2fb111dfd (order=https://acme-v02.api.letsencrypt.org/acme/order/192612760/22920570770) (ca=https://acme-v02.api.letsencrypt.org/directory)","attempt":1,"retrying_in":60,"elapsed":2.650090972,"max_duration":2592000}
{"level":"info","ts":1631101612.867892,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"www.mydomain.com","challenge_type":"dns-01","ca":"https://acme-staging-v02.api.letsencrypt.org/directory"}
{"level":"error","ts":1631101613.3855636,"logger":"tls.issuance.acme.acme_client","msg":"cleaning up solver","identifier":"www.mydomain.com","challenge_type":"dns-01","error":"no memory of presenting a DNS record for www.mydomain.com (probably OK if presenting failed)"}
{"level":"error","ts":1631101613.5276074,"logger":"tls.obtain","msg":"will retry","error":"[www.mydomain.com] Obtain: [www.mydomain.com] solving challenges: presenting for challenge: adding temporary record for zone mydomain.com.: InvalidChangeBatch: InvalidChangeBatch: [Tried to create resource record set [name='_acme-challenge.www.mydomain.com.mydomain.com.', type='TXT'] but it already exists]\n\tstatus code: 400, request id: 8cfb3e00-9657-453d-b867-5e2193a54623 (order=https://acme-staging-v02.api.letsencrypt.org/acme/order/25931808/488611558) (ca=https://acme-staging-v02.api.letsencrypt.org/directory)","attempt":2,"retrying_in":120,"elapsed":64.461055717,"max_duration":2592000}
{"level":"info","ts":1631101735.2945147,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"www.mydomain.com","challenge_type":"dns-01","ca":"https://acme-staging-v02.api.letsencrypt.org/directory"}
{"level":"info","ts":1631101876.6568108,"logger":"tls.issuance.acme.acme_client","msg":"validations succeeded; finalizing order","order":"https://acme-staging-v02.api.letsencrypt.org/acme/order/25931808/488621888"}
{"level":"info","ts":1631101878.9950337,"logger":"tls.issuance.acme.acme_client","msg":"successfully downloaded available certificate chains","count":1,"first_url":"https://acme-staging-v02.api.letsencrypt.org/acme/cert/fa47a21093c8f08654bbf8d740f229b28fdd"}
{"level":"info","ts":1631101878.9953816,"logger":"tls.issuance.acme","msg":"waiting on internal rate limiter","identifiers":["www.mydomain.com"]}
{"level":"info","ts":1631101878.9954185,"logger":"tls.issuance.acme","msg":"done waiting on internal rate limiter","identifiers":["www.mydomain.com"]}
{"level":"info","ts":1631101880.0348005,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"www.mydomain.com","challenge_type":"dns-01","ca":"https://acme-v02.api.letsencrypt.org/directory"}
{"level":"info","ts":1631102028.8770761,"logger":"tls.issuance.acme.acme_client","msg":"validations succeeded; finalizing order","order":"https://acme-v02.api.letsencrypt.org/acme/order/192612760/22921420555"}
{"level":"info","ts":1631102029.8177788,"logger":"tls.issuance.acme.acme_client","msg":"successfully downloaded available certificate chains","count":2,"first_url":"https://acme-v02.api.letsencrypt.org/acme/cert/0412ef9b7a225e28e353c35d0a1119f972cc"}
{"level":"info","ts":1631102029.818293,"logger":"tls.obtain","msg":"certificate obtained successfully","identifier":"www.mydomain.com"}
{"level":"info","ts":1631102029.8183181,"logger":"tls.obtain","msg":"releasing lock","identifier":"www.mydomain.com"}