brajkovic
/
resource_manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: brajkovic:f43afafc7ec56640997bfb3a92d0d9c71e1915af
Branches Tags
brajkovic:master
brajkovic:v0.0.2
brajkovic:v0.0.1
..
compare: brajkovic:d49b8341a11ba5fc5578719d9773c545498ac94e
Branches Tags
brajkovic:master
brajkovic:v0.0.2
brajkovic:v0.0.1

No commits in common. "f43afafc7ec56640997bfb3a92d0d9c71e1915af" and "d49b8341a11ba5fc5578719d9773c545498ac94e" have entirely different histories.
f43afafc7e ... d49b8341a1

13 changed files with 148 additions and 354 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,2 @@
resource_manager
.idea/
.env

43
api/api.go
View File

@ -53,10 +53,6 @@ func HandleUpload(resourceManager resource.Manager) gin.HandlerFunc {
c.AbortWithStatusJSON(400, gin.H{"error": "bad request"})
return
}
if !isValidPath(request.Path) {
c.AbortWithStatusJSON(400, gin.H{"error": "path cannot start with ../ or contain /../"})
return
}
content, err := base64.StdEncoding.DecodeString(request.Content)
if err != nil {
c.AbortWithStatusJSON(400, gin.H{"error": "bad request"})
@ -73,38 +69,26 @@ func HandleUpload(resourceManager resource.Manager) gin.HandlerFunc {
return
}
}
log.Printf("Uploading '%s'...", request.Path)
resourceManager.Upload(resource.UploadRequest{
Buffer: bytes.NewBuffer(content),
Path: request.Path,
MimeType: request.Properties.MimeType,
Overwrite: request.Properties.Overwrite,
})
log.Printf("Uploaded '%s'", request.Path)
// we return this as success
c.Status(http.StatusNoContent)
}
}
func isValidPath(currentPath string) bool {
if strings.HasPrefix(currentPath, "../") || strings.Contains(currentPath, "/../") {
return false
}
return true
}
func HandleCopy(resourceManager resource.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
from := c.Query("from")
to := c.Query("to")
overwrite := c.Query("overwrite") == "true"
log.Printf("Copying from '%s' to '%s' (overwrite=%v)...", from, to, overwrite)
if err := resourceManager.Copy(from, to, overwrite); err != nil {
log.Println(err)
c.AbortWithStatus(500)
} else {
log.Printf("Done copyting from '%s' to '%s' (overwrite=%v)", from, to, overwrite)
c.Status(201)
}
@ -114,17 +98,11 @@ func HandleCopy(resourceManager resource.Manager) gin.HandlerFunc {
func HandlePresign(resourceManager resource.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
path := c.Query("path")
if !isValidPath(path) {
c.AbortWithStatusJSON(400, gin.H{"error": "path cannot start with ../ or contain /../"})
return
}
url, err := resourceManager.Presign(c, path)
if err != nil {
c.AbortWithStatus(404)
return
}
log.Printf("Presigned '%s'", path)
if c.Query("redirect") == "true" {
c.Redirect(http.StatusTemporaryRedirect, url)
} else {
@ -135,22 +113,11 @@ func HandlePresign(resourceManager resource.Manager) gin.HandlerFunc {
func HandleDownload(resourceManager resource.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
path := ""
if c.GetString("secure") == "token" {
path = c.GetString("path")
} else {
path = c.Query("path")
}
if !isValidPath(path) {
c.AbortWithStatusJSON(400, gin.H{"error": "path cannot start with ../ or contain /../"})
return
}
log.Printf("Downloading '%s'...", path)
path := c.Query("path")
data, err := resourceManager.Download(c, path)
if err == nil {
c.Header("content-disposition", "inline; filename=\""+filename(path)+"\"")
c.Data(200, readMimeType(path, ""), data)
log.Printf("Downloaded '%s'", path)
} else {
c.AbortWithStatus(http.StatusNotFound)
}
@ -160,15 +127,9 @@ func HandleDownload(resourceManager resource.Manager) gin.HandlerFunc {
func HandleDelete(resourceManager resource.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
path := c.Query("path")
if !isValidPath(path) {
c.AbortWithStatusJSON(400, gin.H{"error": "path cannot start with ../ or contain /../"})
return
}
log.Printf("Deleting '%s'...", path)
if err := resourceManager.Delete(path); err != nil {
_ = c.AbortWithError(400, err)
c.AbortWithError(400, err)
} else {
log.Printf("Deleted '%s'", path)
c.Status(204)
}
}

79
api/middleware.go
View File

@ -1,79 +0,0 @@
package api
import (
"fmt"
"github.com/gin-gonic/gin"
"log"
"net/http"
"os"
"resource-manager/security"
"strings"
"time"
)
func Auth() gin.HandlerFunc {
basicAuth, hasBasicAuth := os.LookupEnv("BASIC_AUTH_CREDENTIALS")
var apiAuths []string
if list, hasApiAuth := os.LookupEnv("API_CREDENTIALS"); hasApiAuth {
apiAuths = strings.Split(list, ",")
}
return func(c *gin.Context) {
if rawToken := c.Query("token"); rawToken != "" {
token, err := security.ParseToken(rawToken)
if err != nil {
abort(c, nil, http.StatusUnauthorized, "invalid token")
return
}
c.Set("secure", "token")
c.Set("path", token.Path)
return
}
authHeader := c.GetHeader("Authorization")
if strings.HasPrefix(authHeader, "Basic ") && hasBasicAuth {
if strings.TrimPrefix(authHeader, "Basic ") == basicAuth {
c.Set("secure", security.TypeBasic)
return
}
}
if strings.HasPrefix(authHeader, "Api ") && hasBasicAuth {
key := strings.TrimPrefix(authHeader, "Api ")
if hasValidApiKey(apiAuths, key) {
c.Set("secure", security.TypeApi)
return
}
}
abort(c, nil, http.StatusUnauthorized, "missing auth")
}
}
func Secure(types ...security.Type) gin.HandlerFunc {
return func(c *gin.Context) {
value, exists := c.Get("secure")
if !exists {
abort(c, nil, http.StatusUnauthorized, "missing auth")
} else {
securityType := value.(security.Type)
for _, t := range types {
if t == securityType {
return
}
}
abort(c, nil, http.StatusUnauthorized, fmt.Sprintf("bad security: received %s type", securityType))
}
}
}
func hasValidApiKey(auths []string, key string) bool {
for _, a := range auths {
if a == key {
return true
}
}
return false
}
func abort(c *gin.Context, err error, statusCode int, message string) {
log.Printf("Aborted with error: %v", err)
c.AbortWithStatusJSON(statusCode, gin.H{"status": statusCode, "created": time.Now(), "message": message})
}

15
api/routes.go
View File

@ -6,7 +6,6 @@ import (
"os"
"resource-manager/domain/cache"
"resource-manager/domain/resource"
"resource-manager/security"
"time"
)
@ -20,7 +19,7 @@ func createServer() *gin.Engine {
server := gin.New()
server.NoRoute(NoRoute())
server.NoMethod(NoMethod())
server.Use(gin.Recovery(), Auth())
server.Use(gin.Recovery())
return server
}
@ -32,13 +31,11 @@ func RegisterRoutes(server *gin.Engine) {
resourceManager := resource.NewManager(cacheManager, expiration)
resourcesGroup := server.Group("/api/v1/resources", Secure(security.TypeBasic, security.TypeApi))
resourcesGroup.POST("", HandleUpload(resourceManager))
resourcesGroup.GET("presign", HandlePresign(resourceManager))
resourcesGroup.PUT("copy", HandleCopy(resourceManager))
resourcesGroup.DELETE("", HandleDelete(resourceManager))
server.GET("/api/v1/resources", Secure(security.TypeBasic, security.TypeApi, security.TypeToken), HandleDownload(resourceManager))
server.POST("/api/v1/upload", HandleUpload(resourceManager))
server.GET("/api/v1/download", HandleDownload(resourceManager))
server.GET("/api/v1/presign", HandlePresign(resourceManager))
server.PUT("/api/v1/copy", HandleCopy(resourceManager))
server.DELETE("/api/v1/delete", HandleDelete(resourceManager))
}
func loadExpiration() time.Duration {

35
domain/cache/redis_cache.go → domain/cache/impl.go vendored
View File

@ -2,6 +2,7 @@ package cache
import (
"context"
"errors"
"github.com/go-redis/redis/v8"
"os"
"time"
@ -36,3 +37,37 @@ func (r *redisCache) Remove(key string) {
defer cancel()
r.client.Del(ctx, key)
}
func newInMemCache() Manager {
return &inMemCache{make(map[string]inMemEntry)}
}
type inMemEntry struct {
value string
expiration time.Time
}
type inMemCache struct {
entries map[string]inMemEntry
}
func (i *inMemCache) Set(key, value string, expiration time.Duration) error {
i.entries[key] = inMemEntry{
value: value,
expiration: time.Now().Add(expiration),
}
return nil
}
func (i *inMemCache) Get(key string) (string, error) {
if val, exists := i.entries[key]; exists {
if val.expiration.Before(time.Now()) {
delete(i.entries, key)
} else {
return val.value, nil
}
}
return "", errors.New("no value for given key")
}
func (i *inMemCache) Remove(key string) {
delete(i.entries, key)
}

40
domain/cache/in_mem_cache.go vendored
View File

@ -1,40 +0,0 @@
package cache
import (
"errors"
"time"
)
func newInMemCache() Manager {
return &inMemCache{make(map[string]inMemEntry)}
}
type inMemEntry struct {
value string
expiration time.Time
}
type inMemCache struct {
entries map[string]inMemEntry
}
func (i *inMemCache) Set(key, value string, expiration time.Duration) error {
i.entries[key] = inMemEntry{
value: value,
expiration: time.Now().Add(expiration),
}
return nil
}
func (i *inMemCache) Get(key string) (string, error) {
if val, exists := i.entries[key]; exists {
if val.expiration.Before(time.Now()) {
delete(i.entries, key)
} else {
return val.value, nil
}
}
return "", errors.New("no value for given key")
}
func (i *inMemCache) Remove(key string) {
delete(i.entries, key)
}

112
domain/resource/file_manager.go
View File

@ -1,112 +0,0 @@
package resource
import (
"context"
"log"
"os"
"path/filepath"
"resource-manager/domain/cache"
"resource-manager/security"
"strings"
"time"
)
func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manager {
path := os.Getenv("LOCAL_PATH_PREFIX")
// if path isn't set use local
if path == "" {
path, _ = os.Getwd()
}
log.Println("Manager | using path (" + path + ") and expiration of " + expiration.String())
manager := fileManager{cacheManager, path, expiration}
return &manager
}
type fileManager struct {
cache cache.Manager
path string
expiration time.Duration
}
func (f *fileManager) Upload(request UploadRequest) {
fullPath := filepath.Join(f.path, request.Path)
createFolder(fullPath)
if checkFileExists(fullPath) && !request.Overwrite {
log.Println("Manager | cannot upload file as file on same path already exists")
return
}
log.Println("Manager | uploading to (" + request.Path + ")")
if err := os.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil {
log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error())
}
}
func checkFileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func (f *fileManager) Download(_ context.Context, path string) (file []byte, err error) {
fullPath := filepath.Join(f.path, path)
file, err = os.ReadFile(fullPath)
if err != nil {
log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error())
}
return
}
func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
token, err := security.CreateToken(security.PresignToken{Path: path}, f.expiration)
if err != nil {
return "", err
}
return "/api/v1/resources?token=" + token, nil
}
func (f *fileManager) Copy(from string, to string, overwrite bool) error {
fromPath := filepath.Join(f.path, from)
toPath := filepath.Join(f.path, to)
log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")")
createFolder(filepath.Join(toPath, "file"))
return copyFolder(fromPath, toPath, overwrite)
}
func (f *fileManager) Delete(path string) error {
log.Println("Manager | deleting " + path)
fullPath := filepath.Join(f.path, path)
if err := os.Remove(fullPath); err != nil {
log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error())
return err
}
f.cache.Remove(path)
return nil
}
func createFolder(path string) {
paths := strings.Split(path, "/")
folderPath := "/" + filepath.Join(paths[:len(paths)-1]...)
if err := os.MkdirAll(folderPath, 0o755); err != nil {
log.Println("[error] ", err)
}
}
func copyFolder(source, destination string, overwrite bool) error {
var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error {
relPath := strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if d.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
doesExist := false
if _, err := os.Stat(filepath.Join(destination, relPath)); !os.IsNotExist(err) {
doesExist = true
}
if !doesExist || overwrite {
var data, err1 = os.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return os.WriteFile(filepath.Join(destination, relPath), data, 0777)
} else {
return nil
}
}
})
return err
}

111
domain/resource/s3_manager.go → domain/resource/impl.go
View File

@ -8,9 +8,11 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"io/ioutil"
"log"
"net/url"
"os"
"path/filepath"
"resource-manager/domain/cache"
"strconv"
"strings"
@ -18,6 +20,101 @@ import (
"time"
)
func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manager {
path := os.Getenv("UPLOAD_PATH")
// if path isn't set use local
if path == "" {
path, _ = os.Getwd()
}
log.Println("Manager | using local file system for data storage (" + path + ")")
manager := fileManager{cacheManager, path}
return &manager
}
type fileManager struct {
cache cache.Manager
path string
}
func (f *fileManager) Upload(request UploadRequest) {
fullPath := filepath.Join(f.path, request.Path)
createFolder(fullPath)
if checkFileExists(fullPath) && !request.Overwrite {
log.Println("Manager | cannot upload file as file on same path already exists")
return
}
log.Println("Manager | uploading to (" + request.Path + ")")
if err := os.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil {
log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error())
}
}
func checkFileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func (f *fileManager) Download(ctx context.Context, path string) (file []byte, err error) {
fullPath := filepath.Join(f.path, path)
file, err = ioutil.ReadFile(fullPath)
if err != nil {
log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error())
}
return
}
func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
return "/api/v1/get?path=" + path, nil
}
func (f *fileManager) Copy(from string, to string, overwrite bool) error {
fromPath := filepath.Join(f.path, from)
toPath := filepath.Join(f.path, to)
log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")")
createFolder(filepath.Join(toPath, "file"))
return copyFolder(fromPath, toPath, overwrite)
}
func (f *fileManager) Delete(path string) error {
log.Println("Manager | deleting " + path)
fullPath := filepath.Join(f.path, path)
if err := os.Remove(fullPath); err != nil {
log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error())
return err
}
f.cache.Remove(path)
return nil
}
func createFolder(path string) {
paths := strings.Split(path, "/")
folderPath := "/" + filepath.Join(paths[:len(paths)-1]...)
if err := os.MkdirAll(folderPath, 0o755); err != nil {
log.Println("[error] ", err)
}
}
func copyFolder(source, destination string, overwrite bool) error {
var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error {
relPath := strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if d.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
doesExist := false
if _, err := os.Stat(filepath.Join(destination, relPath)); !os.IsNotExist(err) {
doesExist = true
}
if !doesExist || overwrite {
var data, err1 = ioutil.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777)
} else {
return nil
}
}
})
return err
}
func newS3Bucket(cacheManager cache.Manager, expiration time.Duration) Manager {
sess, err := session.NewSession(&aws.Config{
Region: aws.String(os.Getenv("AWS_REGION_NAME"))},
@ -66,22 +163,22 @@ func (a *awsManager) Upload(request UploadRequest) {
a.addToCurrentlyUploading(request.Path)
a.requests <- request
}
func (a *awsManager) Presign(_ context.Context, path string) (string, error) {
func (a *awsManager) Presign(ctx context.Context, path string) (string, error) {
if !a.waitForUploadComplete(path) {
return "", errors.New("failed upload")
}
// ask cache for latest url
if uri, err := a.cache.Get(path); err == nil {
return uri, nil
if url, err := a.cache.Get(path); err == nil {
return url, nil
}
// if there is no value in cache, presign url and add it to cache
uri, err := a.presign(path, a.expiration)
url, err := a.presign(path, a.expiration)
if err != nil {
_ = a.cache.Set(path, uri, a.expiration)
a.cache.Set(path, url, a.expiration)
}
return uri, err
return url, err
}
func (a *awsManager) Download(_ context.Context, path string) (file []byte, err error) {
func (a *awsManager) Download(ctx context.Context, path string) (file []byte, err error) {
return a.download(path)
}
func (a *awsManager) Copy(from string, to string, overwrite bool) error {

1
domain/resource/manager.go
View File

@ -22,7 +22,6 @@ func NewManager(cacheManager cache.Manager, expiration time.Duration) Manager {
log.Println("Manager | using S3 for data storage")
return newS3Bucket(cacheManager, expiration)
} else {
log.Println("Manager | using local file for data storage")
return newLocalFolder(cacheManager, expiration)
}
}

1
go.mod
View File

@ -17,7 +17,6 @@ require (
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect

2
go.sum
View File

@ -33,8 +33,6 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=

2
makefile
View File

@ -1,7 +1,7 @@
# scripts for building account tracker backend
# requires go 1.18+ and git installed
VERSION := 1.1.0
VERSION := 0.0.1
serve:
go run ./...

60
security/generator.go
View File

@ -1,60 +0,0 @@
package security
import (
"errors"
"github.com/golang-jwt/jwt"
"os"
"time"
)
type PresignToken struct {
Path string
}
type Type string
const (
TypeBasic Type = "basic"
TypeApi Type = "api"
TypeToken Type = "token"
)
func CreateToken(tokenInfo PresignToken, duration time.Duration) (string, error) {
// jwt token
atClaims := jwt.MapClaims{}
atClaims["authorized"] = true
// user info
atClaims["path"] = tokenInfo.Path
// expiration
atClaims["exp"] = time.Now().Add(duration).Unix()
at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims)
secret := getSecret()
return at.SignedString([]byte(secret))
}
func ParseToken(token string) (PresignToken, error) {
secret := getSecret()
parsedToken, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) {
return []byte(secret), nil
})
if err != nil {
return PresignToken{}, err
}
claims, ok := parsedToken.Claims.(jwt.MapClaims)
if ok {
return PresignToken{
Path: claims["path"].(string),
}, nil
} else {
return PresignToken{}, errors.New("failed parsing token")
}
}
func getSecret() string {
secret, exists := os.LookupEnv("JWT_SECRET")
if !exists {
panic("env variable JWT_SECRET not set")
}
return secret
}