First commit with real code

This commit is contained in:
2020-08-16 18:47:11 +03:00
parent 40b4f07260
commit c199df5ea6
9 changed files with 419 additions and 0 deletions

12
Dockerfile Normal file
View File

@@ -0,0 +1,12 @@
FROM golang:1.14 AS builder
WORKDIR /root/
RUN GO111MODULE=on go get github.com/minio/minio-go/v7
COPY go.mod ./
COPY *.go ./
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o /root/app .
FROM alpine:latest
RUN apk --no-cache add ca-certificates gnupg
WORKDIR /root/
COPY --from=builder /root/app /root/app
CMD ["/root/app"]

19
docker-compose.yml Normal file
View File

@@ -0,0 +1,19 @@
version: '3'
services:
minio:
image: "minio/minio:latest"
command: ["minio", "server", "/mnt/minio_data"]
ports:
- "9001:9000"
restart: on-failure
env_file: settings.env
environment:
- MINIO_ACCESS_KEY=minio
- MINIO_SECRET_KEY=miniostorage
- MINIO_BROWSER=off
volumes:
- "/tmp/minio_internal:/mnt/minio_data"
gpg_worker:
build: .
restart: on-failure
env_file: settings.env

9
example_settings.env Normal file
View File

@@ -0,0 +1,9 @@
# minio:
# gpg_worker:
TARGET_ENDPOINT=my_vps:9000
TARGET_ACCESS_KEY=ACCESS_KEY
TARGET_SECRET_KEY=SECRET_KEY
TARGET_SSL=true # or false
GPG_KEY_TARGET_BUCKET=SOME_BUCKET_HERE
GPG_KEY_TARGET_NAME=my_public_key.gpg
TARGET_REGION=us-east-1

5
go.mod Normal file
View File

@@ -0,0 +1,5 @@
module s3-gpg-proxy-server
go 1.14
require github.com/minio/minio-go/v7 v7.0.4

66
go.sum Normal file
View File

@@ -0,0 +1,66 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.4 h1:M9glnGclD87VfttesWzURw7SHqq1XDIYGrfTykBTI50=
github.com/minio/minio-go/v7 v7.0.4/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

78
init.go Normal file
View File

@@ -0,0 +1,78 @@
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const ENCRYPT_KEY_PATH = "_key.gpg"
func getMinio(config *S3Config) (*minio.Client, error) {
creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
minioClient, err := minio.New(config.Endpoint, &minio.Options{
Creds: creds,
Secure: config.UseSSL,
Region: config.Region,
})
if err != nil {
return nil, err
}
return minioClient, nil
}
func getInternalMinio(config *AppConfig) (*minio.Client, error) {
return getMinio(&config.LocalS3)
}
func getTargetMinio(config *AppConfig) (*minio.Client, error) {
return getMinio(&config.TargetS3)
}
func prepareKey(config *AppConfig, minioClient *minio.Client) error {
var err error
if _, err := os.Stat(config.Key.LocalPath); !os.IsNotExist(err) {
return nil
}
err = minioClient.FGetObject(
context.Background(),
config.Key.Bucket,
config.Key.Name,
ENCRYPT_KEY_PATH,
minio.GetObjectOptions{},
)
if err == nil {
config.Key.LocalPath = ENCRYPT_KEY_PATH
}
return err
}
func init_app() (*minio.Client, *minio.Client, *AppConfig) {
fmt.Println("initing...")
var err error
config := GetConfig()
var internalMinio *minio.Client
internalMinio, err = getInternalMinio(config)
if err != nil {
log.Fatalln(err)
}
var targetMinio *minio.Client
targetMinio, err = getTargetMinio(config)
if err != nil {
log.Fatalln(err)
}
err = prepareKey(config, targetMinio)
if err != nil {
fmt.Println("Unable to get public key from target S3")
log.Fatalln(err)
}
return internalMinio, targetMinio, config
}

150
main.go Normal file
View File

@@ -0,0 +1,150 @@
package main
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/notification"
)
const TMP_FILE = "_tmp_out"
func processEncryption(
config *AppConfig,
internalMinio *minio.Client,
targetMinio *minio.Client,
bucket string,
name string,
) error {
var err error
found, err := targetMinio.BucketExists(context.Background(), bucket)
if err != nil {
return err
}
if !found {
fmt.Printf("Bucket %v is not exists on remote server, creating...\n", bucket)
err = targetMinio.MakeBucket(
context.Background(),
bucket,
minio.MakeBucketOptions{Region: config.TargetS3.Region},
)
if err != nil {
return err
}
fmt.Printf("Bucket %v has been added on remote server\n", bucket)
}
var objectReader *minio.Object
objectReader, err = internalMinio.GetObject(
context.Background(),
bucket,
name,
minio.GetObjectOptions{},
)
if err != nil {
log.Fatalln(err)
}
cmd := exec.Command("gpg", "--encrypt", "--recipient-file", ENCRYPT_KEY_PATH)
cmd.Stdin = objectReader
var out bytes.Buffer
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
log.Fatalln(err)
}
f, err := os.Create(TMP_FILE)
if err != nil {
log.Fatalln(err)
}
defer f.Close()
f.Write(out.Bytes())
tmp_file, err := os.Open(TMP_FILE)
if err != nil {
log.Fatalln(err)
}
defer tmp_file.Close()
defer os.Remove(TMP_FILE)
uploadInfo, err := targetMinio.PutObject(
context.Background(),
bucket,
name+".gpg",
tmp_file,
-1,
minio.PutObjectOptions{
ContentType: "application/octet-stream",
UserMetadata: map[string]string{"gpg": "true"},
},
)
if err != nil {
fmt.Println(err)
return nil
}
fmt.Println("Successfully uploaded bytes: ", uploadInfo.Size)
err = internalMinio.RemoveObject(
context.Background(),
bucket,
name,
minio.RemoveObjectOptions{},
)
return err
}
func processNotification(
config *AppConfig,
internalMinio *minio.Client,
targetMinio *minio.Client,
event notification.Event,
) error {
var err error
fmt.Printf(
"bucket: %v name: %v size: %v user_meta: %v\n",
event.S3.Bucket.Name,
event.S3.Object.Key,
event.S3.Object.Size,
event.S3.Object.UserMetadata,
)
_, gpg_exists := event.S3.Object.UserMetadata["X-Amz-Meta-Gpg"]
if gpg_exists {
fmt.Println("Already encrypted")
return nil
}
err = processEncryption(
config,
internalMinio,
targetMinio,
event.S3.Bucket.Name,
event.S3.Object.Key,
)
return err
}
func main() {
var err error
internalMinio, targetMinio, config := init_app()
for notificationInfo := range internalMinio.ListenNotification(context.Background(), "", "", []string{
"s3:ObjectCreated:*",
}) {
if notificationInfo.Err != nil {
log.Fatalln(notificationInfo.Err)
}
err = processNotification(config, internalMinio, targetMinio, notificationInfo.Records[0])
if err != nil {
log.Fatalln(err)
}
}
}

BIN
s3-gpg-proxy-server Executable file

Binary file not shown.

80
settings.go Normal file
View File

@@ -0,0 +1,80 @@
package main
import (
"os"
"strconv"
)
type S3Config struct {
Endpoint string
AccessKey string
SecretKey string
UseSSL bool
Region string
}
type GpgKeyConfig struct {
// (Bucket, Name) or LocalPath not empty
Bucket string
Name string
LocalPath string
}
type AppConfig struct {
LocalS3 S3Config
TargetS3 S3Config
Key GpgKeyConfig
DebugMode bool
}
func GetConfig() *AppConfig {
return &AppConfig{
LocalS3: S3Config{
Endpoint: "minio:9000",
AccessKey: "minio",
SecretKey: "miniostorage",
UseSSL: false,
Region: "us-east-1",
},
TargetS3: S3Config{
Endpoint: getEnv("TARGET_ENDPOINT", ""),
AccessKey: getEnv("TARGET_ACCESS_KEY", ""),
SecretKey: getEnv("TARGET_SECRET_KEY", ""),
UseSSL: getEnvAsBool("TARGET_SSL", false),
Region: getEnv("TARGET_REGION", "us-east-1"),
},
Key: GpgKeyConfig{
Bucket: getEnv("GPG_KEY_TARGET_BUCKET", ""),
Name: getEnv("GPG_KEY_TARGET_NAME", ""),
LocalPath: getEnv("GPG_KEY_LOCAL_PATH", ""),
},
DebugMode: getEnvAsBool("DEBUG_MODE", false),
}
}
// Simple helper function to read an environment or return a default value
func getEnv(key string, defaultVal string) string {
if value, exists := os.LookupEnv(key); exists {
return value
}
return defaultVal
}
func getEnvAsInt(name string, defaultVal int) int {
valueStr := getEnv(name, "")
if value, err := strconv.Atoi(valueStr); err == nil {
return value
}
return defaultVal
}
func getEnvAsBool(name string, defaultVal bool) bool {
valStr := getEnv(name, "")
if val, err := strconv.ParseBool(valStr); err == nil {
return val
}
return defaultVal
}