integration test added for clone with client certificate

Author: CrixSlr

test/cmd/lfstest-gitserver.go

@@ -6,16 +6,22 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
+	"crypto/rsa"
 	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"math"
+	"math/big"
 	"net/http"
 	"net/http/httptest"
 	"net/textproto"
@@ -32,10 +38,11 @@ import (
 )
 
 var (
-	repoDir      string
-	largeObjects = newLfsStorage()
-	server       *httptest.Server
-	serverTLS    *httptest.Server
+	repoDir          string
+	largeObjects     = newLfsStorage()
+	server           *httptest.Server
+	serverTLS        *httptest.Server
+	serverClientCert *httptest.Server
 
 	// maps OIDs to content strings. Both the LFS and Storage test servers below
 	// see OIDs.
@@ -61,6 +68,22 @@ func main() {
 	mux := http.NewServeMux()
 	server = httptest.NewServer(mux)
 	serverTLS = httptest.NewTLSServer(mux)
+	serverClientCert = httptest.NewUnstartedServer(mux)
+
+	//setup Client Cert server
+	rootKey, rootCert := generateCARootCertificates()
+	_, clientCertPEM, clientKeyPEM := generateClientCertificates(rootCert, rootKey)
+
+	certPool := x509.NewCertPool()
+	certPool.AddCert(rootCert)
+
+	serverClientCert.TLS = &tls.Config{
+		Certificates: []tls.Certificate{serverTLS.TLS.Certificates[0]},
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		ClientCAs:    certPool,
+	}
+	serverClientCert.StartTLS()
+
 	ntlmSession, err := ntlm.CreateServerSession(ntlm.Version2, ntlm.ConnectionOrientedMode)
 	if err != nil {
 		fmt.Println("Error creating ntlm session:", err)
@@ -108,15 +131,26 @@ func main() {
 	sslurlname := writeTestStateFile([]byte(serverTLS.URL), "LFSTEST_SSL_URL", "lfstest-gitserver-ssl")
 	defer os.RemoveAll(sslurlname)
 
+	clientCertUrlname := writeTestStateFile([]byte(serverClientCert.URL), "LFSTEST_CLIENT_CERT_URL", "lfstest-gitserver-ssl")
+	defer os.RemoveAll(clientCertUrlname)
+
 	block := &pem.Block{}
 	block.Type = "CERTIFICATE"
 	block.Bytes = serverTLS.TLS.Certificates[0].Certificate[0]
 	pembytes := pem.EncodeToMemory(block)
+
 	certname := writeTestStateFile(pembytes, "LFSTEST_CERT", "lfstest-gitserver-cert")
 	defer os.RemoveAll(certname)
 
+	cccertname := writeTestStateFile(clientCertPEM, "LFSTEST_CLIENT_CERT", "lfstest-gitserver-client-cert")
+	defer os.RemoveAll(cccertname)
+
+	ckcertname := writeTestStateFile(clientKeyPEM, "LFSTEST_CLIENT_KEY", "lfstest-gitserver-client-key")
+	defer os.RemoveAll(ckcertname)
+
 	debug("init", "server url: %s", server.URL)
 	debug("init", "server tls url: %s", serverTLS.URL)
+	debug("init", "server client cert url: %s", serverClientCert.URL)
 
 	<-stopch
 	debug("init", "git server done")
@@ -1212,3 +1246,95 @@ func reqId(w http.ResponseWriter) (string, bool) {
 	}
 	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), true
 }
+
+// https://ericchiang.github.io/post/go-tls/
+func generateCARootCertificates() (rootKey *rsa.PrivateKey, rootCert *x509.Certificate) {
+
+	// generate a new key-pair
+	rootKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		log.Fatalf("generating random key: %v", err)
+	}
+
+	rootCertTmpl, err := CertTemplate()
+	if err != nil {
+		log.Fatalf("creating cert template: %v", err)
+	}
+	// describe what the certificate will be used for
+	rootCertTmpl.IsCA = true
+	rootCertTmpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature
+	rootCertTmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
+	//	rootCertTmpl.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")}
+
+	rootCert, _, err = CreateCert(rootCertTmpl, rootCertTmpl, &rootKey.PublicKey, rootKey)
+
+	return
+}
+
+func generateClientCertificates(rootCert *x509.Certificate, rootKey interface{}) (clientKey *rsa.PrivateKey, clientCertPEM []byte, clientKeyPEM []byte) {
+
+	// create a key-pair for the client
+	clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		log.Fatalf("generating random key: %v", err)
+	}
+
+	// create a template for the client
+	clientCertTmpl, err1 := CertTemplate()
+	if err1 != nil {
+		log.Fatalf("creating cert template: %v", err1)
+	}
+	clientCertTmpl.KeyUsage = x509.KeyUsageDigitalSignature
+	clientCertTmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+
+	// the root cert signs the cert by again providing its private key
+	_, clientCertPEM, err2 := CreateCert(clientCertTmpl, rootCert, &clientKey.PublicKey, rootKey)
+	if err2 != nil {
+		log.Fatalf("error creating cert: %v", err2)
+	}
+
+	// encode and load the cert and private key for the client
+	clientKeyPEM = pem.EncodeToMemory(&pem.Block{
+		Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(clientKey),
+	})
+
+	return
+}
+
+// helper function to create a cert template with a serial number and other required fields
+func CertTemplate() (*x509.Certificate, error) {
+	// generate a random serial number (a real cert authority would have some logic behind this)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return nil, errors.New("failed to generate serial number: " + err.Error())
+	}
+
+	tmpl := x509.Certificate{
+		SerialNumber:          serialNumber,
+		Subject:               pkix.Name{Organization: []string{"Yhat, Inc."}},
+		SignatureAlgorithm:    x509.SHA256WithRSA,
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour), // valid for an hour
+		BasicConstraintsValid: true,
+	}
+	return &tmpl, nil
+}
+
+func CreateCert(template, parent *x509.Certificate, pub interface{}, parentPriv interface{}) (
+	cert *x509.Certificate, certPEM []byte, err error) {
+
+	certDER, err := x509.CreateCertificate(rand.Reader, template, parent, pub, parentPriv)
+	if err != nil {
+		return
+	}
+	// parse the resulting certificate so we can use it again
+	cert, err = x509.ParseCertificate(certDER)
+	if err != nil {
+		return
+	}
+	// PEM encode the certificate (this is a standard TLS encoding)
+	b := pem.Block{Type: "CERTIFICATE", Bytes: certDER}
+	certPEM = pem.EncodeToMemory(&b)
+	return
+}

test/test-clone.sh

@@ -140,6 +140,66 @@ begin_test "cloneSSL"
 )
 end_test
 
+begin_test "clone ClientCert"
+(
+
+  set -e
+#  if $TRAVIS; then
+#    echo "Skipping SSL tests, Travis has weird behaviour in validating custom certs, test locally only"
+#    exit 0
+#  fi
+
+  reponame="test-cloneClientCert"
+  setup_remote_repo "$reponame"
+  clone_repo_clientcert "$reponame" "$reponame"
+
+  git lfs track "*.dat" 2>&1 | tee track.log
+  grep "Tracking \*.dat" track.log
+
+  # generate some test data & commits with random LFS data
+  echo "[
+  {
+    \"CommitDate\":\"$(get_date -5d)\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":100},
+      {\"Filename\":\"file2.dat\",\"Size\":75}]
+  },
+  {
+    \"CommitDate\":\"$(get_date -1d)\",
+    \"Files\":[
+      {\"Filename\":\"file3.dat\",\"Size\":30}]
+  }
+  ]" | lfstest-testutils addcommits
+
+  git push origin master
+
+  # Now clone again with 'git lfs clone', test specific clone dir
+  cd "$TRASHDIR"
+
+  newclonedir="testcloneClietCert1"
+  git lfs clone "$CLIENTCERTGITSERVER/$reponame" "$newclonedir" 2>&1 | tee lfsclone.log
+  grep "Cloning into" lfsclone.log
+  grep "Git LFS:" lfsclone.log
+  # should be no filter errors
+  [ ! $(grep "filter" lfsclone.log) ]
+  [ ! $(grep "error" lfsclone.log) ]
+  # should be cloned into location as per arg
+  [ -d "$newclonedir" ]
+
+  # check a few file sizes to make sure pulled
+  pushd "$newclonedir"
+  [ $(wc -c < "file1.dat") -eq 100 ]
+  [ $(wc -c < "file2.dat") -eq 75 ]
+  [ $(wc -c < "file3.dat") -eq 30 ]
+  popd
+
+
+  # Now check SSL clone with standard 'git clone' and smudge download
+  rm -rf "$reponame"
+  git clone "$CLIENTCERTGITSERVER/$reponame"
+
+)
+end_test
 
 begin_test "clone with flags"
 (

test/testenv.sh

@@ -102,9 +102,19 @@ LFS_URL_FILE="$REMOTEDIR/url"
 # section in test/README.md
 LFS_SSL_URL_FILE="$REMOTEDIR/sslurl"
 
+# This file contains the client cert SSL URL of the test Git server. See the "Test Suite"
+# section in test/README.md
+LFS_CLIENT_CERT_URL_FILE="$REMOTEDIR/clientcerturl"
+
 # This file contains the self-signed SSL cert of the TLS endpoint of the test Git server.
 LFS_CERT_FILE="$REMOTEDIR/cert"
 
+# This file contains the client certificate of the client cert endpoint of the test Git server.
+LFS_CLIENT_CERT_FILE="$REMOTEDIR/client.crt"
+
+# This file contains the client key of the client cert endpoint of the test Git server.
+LFS_CLIENT_KEY_FILE="$REMOTEDIR/client.key"
+
 # the fake home dir used for the initial setup
 TESTHOME="$REMOTEDIR/home"
 

test/testhelpers.sh

@@ -258,6 +258,25 @@ clone_repo_ssl() {
   echo "$out"
 }
 
+# clone_repo_clientcert clones a repository from the test Git server to the subdirectory
+# $dir under $TRASHDIR, using the client cert endpoint.
+# setup_remote_repo() needs to be run first. Output is written to clone_client_cert.log.
+clone_repo_clientcert() {
+  cd "$TRASHDIR"
+
+  local reponame="$1"
+  local dir="$2"
+  echo "clone $CLIENTCERTGITSERVER/$reponame to $dir"
+  out=$(git clone "$CLIENTCERTGITSERVER/$reponame" "$dir" 2>&1)
+  cd "$dir"
+
+  git config credential.helper lfstest
+#todo setup client cert...
+
+  echo "$out" > clone_client_cert.log
+  echo "$out"
+}
+
 # setup_remote_repo_with_file creates a remote repo, clones it locally, commits
 # a file tracked by LFS, and pushes it to the remote:
 #
@@ -330,7 +349,16 @@ setup() {
     fi
   fi
 
-  LFSTEST_URL="$LFS_URL_FILE" LFSTEST_SSL_URL="$LFS_SSL_URL_FILE" LFSTEST_DIR="$REMOTEDIR" LFSTEST_CERT="$LFS_CERT_FILE" lfstest-gitserver > "$REMOTEDIR/gitserver.log" 2>&1 &
+  LFSTEST_URL="$LFS_URL_FILE" LFSTEST_SSL_URL="$LFS_SSL_URL_FILE" LFSTEST_CLIENT_CERT_URL="$LFS_CLIENT_CERT_URL_FILE" LFSTEST_DIR="$REMOTEDIR" LFSTEST_CERT="$LFS_CERT_FILE" LFSTEST_CLIENT_CERT="$LFS_CLIENT_CERT_FILE" LFSTEST_CLIENT_KEY="$LFS_CLIENT_KEY_FILE" lfstest-gitserver > "$REMOTEDIR/gitserver.log" 2>&1 &
+
+  wait_for_file "$LFS_URL_FILE"
+  wait_for_file "$LFS_SSL_URL_FILE"
+  wait_for_file "$LFS_CLIENT_CERT_URL_FILE"
+  wait_for_file "$LFS_CERT_FILE"
+  wait_for_file "$LFS_CLIENT_CERT_FILE"
+  wait_for_file "$LFS_CLIENT_KEY_FILE"
+
+  LFS_CLIENT_CERT_URL=`cat $LFS_CLIENT_CERT_URL_FILE`
 
   # Set up the initial git config and osx keychain if applicable
   HOME="$TESTHOME"
@@ -341,6 +369,8 @@ setup() {
   git config --global user.name "Git LFS Tests"
   git config --global user.email "[email protected]"
   git config --global http.sslcainfo "$LFS_CERT_FILE"
+  git config --global http.$LFS_CLIENT_CERT_URL/.sslKey "$LFS_CLIENT_KEY_FILE"
+  git config --global http.$LFS_CLIENT_CERT_URL/.sslCert "$LFS_CLIENT_CERT_FILE"
 
   ( grep "git-lfs clean" "$REMOTEDIR/home/.gitconfig" > /dev/null && grep "git-lfs filter-process" "$REMOTEDIR/home/.gitconfig" > /dev/null ) || {
     echo "global git config should be set in $REMOTEDIR/home"
@@ -359,15 +389,14 @@ setup() {
   echo "lfstest-gitserver:"
   echo "  LFSTEST_URL=$LFS_URL_FILE"
   echo "  LFSTEST_SSL_URL=$LFS_SSL_URL_FILE"
+  echo "  LFSTEST_CLIENT_CERT_URL=$LFS_CLIENT_CERT_URL_FILE ($LFS_CLIENT_CERT_URL)"
   echo "  LFSTEST_CERT=$LFS_CERT_FILE"
+  echo "  LFSTEST_CLIENT_CERT=$LFS_CLIENT_CERT_FILE"
+  echo "  LFSTEST_CLIENT_KEY=$LFS_CLIENT_KEY_FILE"
   echo "  LFSTEST_DIR=$REMOTEDIR"
   echo "GIT:"
   git config --global --get-regexp "lfs|credential|user"
 
-  wait_for_file "$LFS_URL_FILE"
-  wait_for_file "$LFS_SSL_URL_FILE"
-  wait_for_file "$LFS_CERT_FILE"
-
   echo
 }
 

test/testlib.sh

@@ -59,6 +59,7 @@ fi
 
 GITSERVER=$(cat "$LFS_URL_FILE")
 SSLGITSERVER=$(cat "$LFS_SSL_URL_FILE")
+CLIENTCERTGITSERVER=$(cat "$LFS_CLIENT_CERT_URL_FILE")
 cd "$TRASHDIR"
 
 # Mark the beginning of a test. A subshell should immediately follow this