Let's Encrypt Certificate Generation (#327)

* Manual rebase of get-certs branch

* fix endpoints, add verbose flag

* more stable pre-check behaviour

* start of docs

* docs for get-certs

* don't require cert for dnscontrol

* fix up directory paths

* small doc tweaks

pkg/acme/acme.go

@@ -0,0 +1,290 @@
+// Package acme provides a means of performing Let's Encrypt DNS challenges via a DNSConfig
+package acme
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/StackExchange/dnscontrol/models"
+	"github.com/StackExchange/dnscontrol/pkg/nameservers"
+	"github.com/xenolf/lego/acmev2"
+)
+
+type CertConfig struct {
+	CertName string   `json:"cert_name"`
+	Names    []string `json:"names"`
+}
+
+type Client interface {
+	IssueOrRenewCert(config *CertConfig, renewUnder int, verbose bool) (bool, error)
+}
+
+type certManager struct {
+	directory      string
+	email          string
+	acmeDirectory  string
+	acmeHost       string
+	cfg            *models.DNSConfig
+	checkedDomains map[string]bool
+
+	account *account
+	client  *acme.Client
+}
+
+const (
+	LetsEncryptLive  = "https://acme-v02.api.letsencrypt.org/directory"
+	LetsEncryptStage = "https://acme-staging-v02.api.letsencrypt.org/directory"
+)
+
+func New(cfg *models.DNSConfig, directory string, email string, server string) (Client, error) {
+	u, err := url.Parse(server)
+	if err != nil || u.Host == "" {
+		return nil, fmt.Errorf("ACME directory '%s' is not a valid URL", server)
+	}
+	c := &certManager{
+		directory:      directory,
+		email:          email,
+		acmeDirectory:  server,
+		acmeHost:       u.Host,
+		cfg:            cfg,
+		checkedDomains: map[string]bool{},
+	}
+
+	if err := c.loadOrCreateAccount(); err != nil {
+		return nil, err
+	}
+	c.client.ExcludeChallenges([]acme.Challenge{acme.HTTP01})
+	c.client.SetChallengeProvider(acme.DNS01, c)
+	return c, nil
+}
+
+// IssueOrRenewCert will obtain a certificate with the given name if it does not exist,
+// or renew it if it is close enough to the expiration date.
+// It will return true if it issued or updated the certificate.
+func (c *certManager) IssueOrRenewCert(cfg *CertConfig, renewUnder int, verbose bool) (bool, error) {
+	if !verbose {
+		acme.Logger = log.New(ioutil.Discard, "", 0)
+	}
+
+	log.Printf("Checking certificate [%s]", cfg.CertName)
+	if err := os.MkdirAll(filepath.Dir(c.certFile(cfg.CertName, "json")), perms); err != nil {
+		return false, err
+	}
+	existing, err := c.readCertificate(cfg.CertName)
+	if err != nil {
+		return false, err
+	}
+
+	var action = func() (acme.CertificateResource, error) {
+		return c.client.ObtainCertificate(cfg.Names, true, nil, true)
+	}
+
+	if existing == nil {
+		log.Println("No existing cert found. Issuing new...")
+	} else {
+		names, daysLeft, err := getCertInfo(existing.Certificate)
+		if err != nil {
+			return false, err
+		}
+		log.Printf("Found existing cert. %0.2f days remaining.", daysLeft)
+		namesOK := dnsNamesEqual(cfg.Names, names)
+		if daysLeft >= float64(renewUnder) && namesOK {
+			log.Println("Nothing to do")
+			//nothing to do
+			return false, nil
+		}
+		if !namesOK {
+			log.Println("DNS Names don't match expected set. Reissuing.")
+		} else {
+			log.Println("Renewing cert")
+			action = func() (acme.CertificateResource, error) {
+				return c.client.RenewCertificate(*existing, true, true)
+			}
+		}
+	}
+
+	certResource, err := action()
+	if err != nil {
+		return false, err
+	}
+	fmt.Printf("Obtained certificate for %s\n", cfg.CertName)
+	return true, c.writeCertificate(cfg.CertName, &certResource)
+}
+
+// filename for certifiacte / key / json file
+func (c *certManager) certFile(name, ext string) string {
+	return filepath.Join(c.directory, "certificates", name, name+"."+ext)
+}
+
+func (c *certManager) writeCertificate(name string, cr *acme.CertificateResource) error {
+	jDAt, err := json.MarshalIndent(cr, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(c.certFile(name, "json"), jDAt, perms); err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(c.certFile(name, "crt"), cr.Certificate, perms); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(c.certFile(name, "key"), cr.PrivateKey, perms)
+}
+
+func getCertInfo(pemBytes []byte) (names []string, remaining float64, err error) {
+	block, _ := pem.Decode(pemBytes)
+	if block == nil {
+		return nil, 0, fmt.Errorf("Invalid certificate pem data")
+	}
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, 0, err
+	}
+	var daysLeft = float64(cert.NotAfter.Sub(time.Now())) / float64(time.Hour*24)
+	return cert.DNSNames, daysLeft, nil
+}
+
+// checks two lists of sans to make sure they have all the same names in them.
+func dnsNamesEqual(a []string, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	sort.Strings(a)
+	sort.Strings(b)
+	for i, s := range a {
+		if b[i] != s {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *certManager) readCertificate(name string) (*acme.CertificateResource, error) {
+	f, err := os.Open(c.certFile(name, "json"))
+	if err != nil && os.IsNotExist(err) {
+		// if json does not exist, nothing does
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	dec := json.NewDecoder(f)
+	cr := &acme.CertificateResource{}
+	if err = dec.Decode(cr); err != nil {
+		return nil, err
+	}
+	// load cert
+	crtBytes, err := ioutil.ReadFile(c.certFile(name, "crt"))
+	if err != nil {
+		return nil, err
+	}
+	cr.Certificate = crtBytes
+	return cr, nil
+}
+
+func (c *certManager) Present(domain, token, keyAuth string) (e error) {
+	d := c.cfg.DomainContainingFQDN(domain)
+	// fix NS records for this domain's DNS providers
+	// only need to do this once per domain
+	const metaKey = "x-fixed-nameservers"
+	if d.Metadata[metaKey] == "" {
+		nsList, err := nameservers.DetermineNameservers(d)
+		if err != nil {
+			return err
+		}
+		d.Nameservers = nsList
+		nameservers.AddNSRecords(d)
+		d.Metadata[metaKey] = "true"
+	}
+	// copy now so we can add txt record safely, and just run unmodified version later to cleanup
+	d, err := d.Copy()
+	if err != nil {
+		return err
+	}
+	if err := c.ensureNoPendingCorrections(d); err != nil {
+		return err
+	}
+	fqdn, val, _ := acme.DNS01Record(domain, keyAuth)
+	fmt.Println(fqdn, val)
+	txt := &models.RecordConfig{Type: "TXT"}
+	txt.SetTargetTXT(val)
+	txt.SetLabelFromFQDN(fqdn, d.Name)
+	d.Records = append(d.Records, txt)
+
+	return getAndRunCorrections(d)
+}
+
+func (c *certManager) ensureNoPendingCorrections(d *models.DomainConfig) error {
+	// only need to check a domain once per app run
+	if c.checkedDomains[d.Name] {
+		return nil
+	}
+	corrections, err := getCorrections(d)
+	if err != nil {
+		return err
+	}
+	if len(corrections) != 0 {
+		// TODO: maybe allow forcing through this check.
+		for _, c := range corrections {
+			fmt.Println(c.Msg)
+		}
+		return fmt.Errorf("Found %d pending corrections for %s. Not going to proceed issuing certificates", len(corrections), d.Name)
+	}
+	return nil
+}
+
+// IgnoredProviders is a lit of provider names that should not be used to fill challenges.
+var IgnoredProviders = map[string]bool{}
+
+func getCorrections(d *models.DomainConfig) ([]*models.Correction, error) {
+	cs := []*models.Correction{}
+	for _, p := range d.DNSProviderInstances {
+		if IgnoredProviders[p.Name] {
+			continue
+		}
+		dc, err := d.Copy()
+		if err != nil {
+			return nil, err
+		}
+		corrections, err := p.Driver.GetDomainCorrections(dc)
+		if err != nil {
+			return nil, err
+		}
+		for _, c := range corrections {
+			c.Msg = fmt.Sprintf("[%s] %s", p.Name, strings.TrimSpace(c.Msg))
+		}
+		cs = append(cs, corrections...)
+	}
+	return cs, nil
+}
+
+func getAndRunCorrections(d *models.DomainConfig) error {
+	cs, err := getCorrections(d)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%d corrections\n", len(cs))
+	for _, c := range cs {
+		fmt.Printf("Running [%s]\n", c.Msg)
+		err = c.F()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *certManager) CleanUp(domain, token, keyAuth string) error {
+	d := c.cfg.DomainContainingFQDN(domain)
+	return getAndRunCorrections(d)
+}

pkg/acme/checkDns.go

@@ -0,0 +1,31 @@
+package acme
+
+import (
+	"log"
+	"time"
+
+	"github.com/xenolf/lego/acmev2"
+)
+
+func init() {
+	// default record verification in the client library makes sure the authoritative nameservers
+	// have the expected records.
+	// Sometimes the Let's Encrypt verification fails anyway because records have not propagated the provider's network fully.
+	// So we add an additional 20 second sleep just for safety.
+	origCheck := acme.PreCheckDNS
+	acme.PreCheckDNS = func(fqdn, value string) (bool, error) {
+		start := time.Now()
+		v, err := origCheck(fqdn, value)
+		if err != nil {
+			return v, err
+		}
+		log.Printf("DNS ok after %s. Waiting again for propagation", time.Now().Sub(start))
+		time.Sleep(20 * time.Second)
+		return v, err
+	}
+}
+
+// Timeout increases the client-side polling check time to five minutes with one second waits in-between.
+func (c *certManager) Timeout() (timeout, interval time.Duration) {
+	return 5 * time.Minute, time.Second
+}

pkg/acme/registration.go

@@ -0,0 +1,121 @@
+package acme
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/xenolf/lego/acmev2"
+)
+
+func (c *certManager) loadOrCreateAccount() error {
+	f, err := os.Open(c.accountFile())
+	if err != nil && os.IsNotExist(err) {
+		return c.createAccount()
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	dec := json.NewDecoder(f)
+	acct := &account{}
+	if err = dec.Decode(acct); err != nil {
+		return err
+	}
+	c.account = acct
+	keyBytes, err := ioutil.ReadFile(c.accountKeyFile())
+	if err != nil {
+		return err
+	}
+	keyBlock, _ := pem.Decode(keyBytes)
+	if keyBlock == nil {
+		log.Fatal("WTF", keyBytes)
+	}
+	c.account.key, err = x509.ParseECPrivateKey(keyBlock.Bytes)
+	if err != nil {
+		return err
+	}
+	c.client, err = acme.NewClient(c.acmeDirectory, c.account, acme.RSA2048) // TODO: possibly make configurable on a cert-by cert basis
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *certManager) accountDirectory() string {
+	return filepath.Join(c.directory, ".letsencrypt", c.acmeHost)
+}
+
+func (c *certManager) accountFile() string {
+	return filepath.Join(c.accountDirectory(), "account.json")
+}
+func (c *certManager) accountKeyFile() string {
+	return filepath.Join(c.accountDirectory(), "account.key")
+}
+
+const perms os.FileMode = 0644 // TODO: probably lock this down more
+
+func (c *certManager) createAccount() error {
+	if err := os.MkdirAll(c.accountDirectory(), perms); err != nil {
+		return err
+	}
+	privateKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	if err != nil {
+		return err
+	}
+	acct := &account{
+		key:   privateKey,
+		Email: c.email,
+	}
+	c.account = acct
+	c.client, err = acme.NewClient(c.acmeDirectory, c.account, acme.EC384)
+	if err != nil {
+		return err
+	}
+	reg, err := c.client.Register(true)
+	if err != nil {
+		return err
+	}
+	c.account.Registration = reg
+	acctBytes, err := json.MarshalIndent(c.account, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(c.accountFile(), acctBytes, perms); err != nil {
+		return err
+	}
+	keyBytes, err := x509.MarshalECPrivateKey(privateKey)
+	if err != nil {
+		return err
+	}
+	pemKey := &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}
+	pemBytes := pem.EncodeToMemory(pemKey)
+	if err = ioutil.WriteFile(c.accountKeyFile(), pemBytes, perms); err != nil {
+		return err
+	}
+	return nil
+}
+
+type account struct {
+	Email        string `json:"email"`
+	key          crypto.PrivateKey
+	Registration *acme.RegistrationResource `json:"registration"`
+}
+
+func (a *account) GetEmail() string {
+	return a.Email
+}
+func (a *account) GetPrivateKey() crypto.PrivateKey {
+	return a.key
+}
+func (a *account) GetRegistration() *acme.RegistrationResource {
+	return a.Registration
+}