diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b7482cb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.14 AS builder +WORKDIR /root/ +RUN GO111MODULE=on go get github.com/minio/minio-go/v7 +COPY go.mod ./ +COPY *.go ./ +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o /root/app . + +FROM alpine:latest +RUN apk --no-cache add ca-certificates gnupg +WORKDIR /root/ +COPY --from=builder /root/app /root/app +CMD ["/root/app"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8923d2f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + minio: + image: "minio/minio:latest" + command: ["minio", "server", "/mnt/minio_data"] + ports: + - "9001:9000" + restart: on-failure + env_file: settings.env + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=miniostorage + - MINIO_BROWSER=off + volumes: + - "/tmp/minio_internal:/mnt/minio_data" + gpg_worker: + build: . + restart: on-failure + env_file: settings.env diff --git a/example_settings.env b/example_settings.env new file mode 100644 index 0000000..ea28bdd --- /dev/null +++ b/example_settings.env @@ -0,0 +1,9 @@ +# minio: +# gpg_worker: +TARGET_ENDPOINT=my_vps:9000 +TARGET_ACCESS_KEY=ACCESS_KEY +TARGET_SECRET_KEY=SECRET_KEY +TARGET_SSL=true # or false +GPG_KEY_TARGET_BUCKET=SOME_BUCKET_HERE +GPG_KEY_TARGET_NAME=my_public_key.gpg +TARGET_REGION=us-east-1 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cca57f3 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module s3-gpg-proxy-server + +go 1.14 + +require github.com/minio/minio-go/v7 v7.0.4 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3944488 --- /dev/null +++ b/go.sum @@ -0,0 +1,66 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.4 h1:M9glnGclD87VfttesWzURw7SHqq1XDIYGrfTykBTI50= +github.com/minio/minio-go/v7 v7.0.4/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/init.go b/init.go new file mode 100644 index 0000000..1147c07 --- /dev/null +++ b/init.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const ENCRYPT_KEY_PATH = "_key.gpg" + +func getMinio(config *S3Config) (*minio.Client, error) { + creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "") + minioClient, err := minio.New(config.Endpoint, &minio.Options{ + Creds: creds, + Secure: config.UseSSL, + Region: config.Region, + }) + if err != nil { + return nil, err + } + return minioClient, nil +} + +func getInternalMinio(config *AppConfig) (*minio.Client, error) { + return getMinio(&config.LocalS3) +} + +func getTargetMinio(config *AppConfig) (*minio.Client, error) { + return getMinio(&config.TargetS3) +} + +func prepareKey(config *AppConfig, minioClient *minio.Client) error { + var err error + if _, err := os.Stat(config.Key.LocalPath); !os.IsNotExist(err) { + return nil + } + err = minioClient.FGetObject( + context.Background(), + config.Key.Bucket, + config.Key.Name, + ENCRYPT_KEY_PATH, + minio.GetObjectOptions{}, + ) + if err == nil { + config.Key.LocalPath = ENCRYPT_KEY_PATH + } + return err +} + +func init_app() (*minio.Client, *minio.Client, *AppConfig) { + fmt.Println("initing...") + var err error + + config := GetConfig() + + var internalMinio *minio.Client + internalMinio, err = getInternalMinio(config) + if err != nil { + log.Fatalln(err) + } + + var targetMinio *minio.Client + targetMinio, err = getTargetMinio(config) + if err != nil { + log.Fatalln(err) + } + + err = prepareKey(config, targetMinio) + if err != nil { + fmt.Println("Unable to get public key from target S3") + log.Fatalln(err) + } + return internalMinio, targetMinio, config +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e8fb391 --- /dev/null +++ b/main.go @@ -0,0 +1,150 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + + "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/notification" +) + +const TMP_FILE = "_tmp_out" + +func processEncryption( + config *AppConfig, + internalMinio *minio.Client, + targetMinio *minio.Client, + bucket string, + name string, +) error { + var err error + + found, err := targetMinio.BucketExists(context.Background(), bucket) + if err != nil { + return err + } + if !found { + fmt.Printf("Bucket %v is not exists on remote server, creating...\n", bucket) + err = targetMinio.MakeBucket( + context.Background(), + bucket, + minio.MakeBucketOptions{Region: config.TargetS3.Region}, + ) + if err != nil { + return err + } + fmt.Printf("Bucket %v has been added on remote server\n", bucket) + } + + var objectReader *minio.Object + objectReader, err = internalMinio.GetObject( + context.Background(), + bucket, + name, + minio.GetObjectOptions{}, + ) + + if err != nil { + log.Fatalln(err) + } + + cmd := exec.Command("gpg", "--encrypt", "--recipient-file", ENCRYPT_KEY_PATH) + cmd.Stdin = objectReader + var out bytes.Buffer + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + log.Fatalln(err) + } + + f, err := os.Create(TMP_FILE) + if err != nil { + log.Fatalln(err) + } + + defer f.Close() + f.Write(out.Bytes()) + + tmp_file, err := os.Open(TMP_FILE) + if err != nil { + log.Fatalln(err) + } + defer tmp_file.Close() + defer os.Remove(TMP_FILE) + + uploadInfo, err := targetMinio.PutObject( + context.Background(), + bucket, + name+".gpg", + tmp_file, + -1, + minio.PutObjectOptions{ + ContentType: "application/octet-stream", + UserMetadata: map[string]string{"gpg": "true"}, + }, + ) + if err != nil { + fmt.Println(err) + return nil + } + fmt.Println("Successfully uploaded bytes: ", uploadInfo.Size) + + err = internalMinio.RemoveObject( + context.Background(), + bucket, + name, + minio.RemoveObjectOptions{}, + ) + return err +} + +func processNotification( + config *AppConfig, + internalMinio *minio.Client, + targetMinio *minio.Client, + event notification.Event, +) error { + var err error + fmt.Printf( + "bucket: %v name: %v size: %v user_meta: %v\n", + event.S3.Bucket.Name, + event.S3.Object.Key, + event.S3.Object.Size, + event.S3.Object.UserMetadata, + ) + _, gpg_exists := event.S3.Object.UserMetadata["X-Amz-Meta-Gpg"] + if gpg_exists { + fmt.Println("Already encrypted") + return nil + } + err = processEncryption( + config, + internalMinio, + targetMinio, + event.S3.Bucket.Name, + event.S3.Object.Key, + ) + return err +} + +func main() { + var err error + internalMinio, targetMinio, config := init_app() + + for notificationInfo := range internalMinio.ListenNotification(context.Background(), "", "", []string{ + "s3:ObjectCreated:*", + }) { + if notificationInfo.Err != nil { + log.Fatalln(notificationInfo.Err) + } + err = processNotification(config, internalMinio, targetMinio, notificationInfo.Records[0]) + if err != nil { + log.Fatalln(err) + } + } +} diff --git a/s3-gpg-proxy-server b/s3-gpg-proxy-server new file mode 100755 index 0000000..46fdde7 Binary files /dev/null and b/s3-gpg-proxy-server differ diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..9218b87 --- /dev/null +++ b/settings.go @@ -0,0 +1,80 @@ +package main + +import ( + "os" + "strconv" +) + +type S3Config struct { + Endpoint string + AccessKey string + SecretKey string + UseSSL bool + Region string +} + +type GpgKeyConfig struct { + // (Bucket, Name) or LocalPath not empty + Bucket string + Name string + LocalPath string +} + +type AppConfig struct { + LocalS3 S3Config + TargetS3 S3Config + Key GpgKeyConfig + DebugMode bool +} + +func GetConfig() *AppConfig { + return &AppConfig{ + LocalS3: S3Config{ + Endpoint: "minio:9000", + AccessKey: "minio", + SecretKey: "miniostorage", + UseSSL: false, + Region: "us-east-1", + }, + TargetS3: S3Config{ + Endpoint: getEnv("TARGET_ENDPOINT", ""), + AccessKey: getEnv("TARGET_ACCESS_KEY", ""), + SecretKey: getEnv("TARGET_SECRET_KEY", ""), + UseSSL: getEnvAsBool("TARGET_SSL", false), + Region: getEnv("TARGET_REGION", "us-east-1"), + }, + Key: GpgKeyConfig{ + Bucket: getEnv("GPG_KEY_TARGET_BUCKET", ""), + Name: getEnv("GPG_KEY_TARGET_NAME", ""), + LocalPath: getEnv("GPG_KEY_LOCAL_PATH", ""), + }, + DebugMode: getEnvAsBool("DEBUG_MODE", false), + } +} + +// Simple helper function to read an environment or return a default value +func getEnv(key string, defaultVal string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + + return defaultVal +} + +func getEnvAsInt(name string, defaultVal int) int { + valueStr := getEnv(name, "") + if value, err := strconv.Atoi(valueStr); err == nil { + return value + } + + return defaultVal +} + +func getEnvAsBool(name string, defaultVal bool) bool { + valStr := getEnv(name, "") + if val, err := strconv.ParseBool(valStr); err == nil { + return val + } + + return defaultVal +}