brajkovic
/
resource_manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implemented presign logic for file manager

This commit is contained in:
Borna Rajković 2024-01-06 14:10:24 +01:00
parent fdd46f5086
commit 20cc4db097
11 changed files with 234 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/api.go
View File

@ -120,13 +120,18 @@ func HandlePresign(resourceManager resource.Manager) gin.HandlerFunc {
func HandleDownload(resourceManager resource.Manager) gin.HandlerFunc {
return func(c *gin.Context) {
path := c.Query("path")
path := ""
if c.GetString("secure") == "token" {
path = c.GetString("path")
} else {
path = c.Query("path")
}
log.Printf("Downloading '%s'...", path)
data, err := resourceManager.Download(c, path)
if err == nil {
log.Printf("Downloaded '%s'", path)
c.Header("content-disposition", "inline; filename=\""+filename(path)+"\"")
c.Data(200, readMimeType(path, ""), data)
log.Printf("Downloaded '%s'", path)
} else {
c.AbortWithStatus(http.StatusNotFound)
}

13
api/middleware.go
View File

@ -6,6 +6,7 @@ import (
"log"
"net/http"
"os"
"resource-manager/security"
"strings"
"time"
)
@ -18,6 +19,16 @@ func Auth() gin.HandlerFunc {
}
return func(c *gin.Context) {
if rawToken := c.Query("token"); rawToken != "" {
token, err := security.ParseToken(rawToken)
if err != nil {
abort(c, nil, http.StatusUnauthorized, "invalid token")
return
}
c.Set("secure", "token")
c.Set("path", token.Path)
return
}
authHeader := c.GetHeader("Authorization")
if strings.HasPrefix(authHeader, "Basic ") && hasBasicAuth {
if strings.TrimPrefix(authHeader, "Basic ") == basicAuth {
@ -63,6 +74,6 @@ func hasValidApiKey(auths []string, key string) bool {
}
func abort(c *gin.Context, err error, statusCode int, message string) {
log.Printf("Aborted: %v", err.Error())
log.Printf("Aborted with error: %v", err)
c.AbortWithStatusJSON(statusCode, gin.H{"status": statusCode, "created": time.Now(), "message": message})
}

12
api/routes.go
View File

@ -31,13 +31,13 @@ func RegisterRoutes(server *gin.Engine) {
resourceManager := resource.NewManager(cacheManager, expiration)
v1Group := server.Group("/api/v1/resources", Secure("basic", "api"))
resourcesGroup := server.Group("/api/v1/resources", Secure("basic", "api"))
resourcesGroup.POST("", HandleUpload(resourceManager))
resourcesGroup.GET("presign", HandlePresign(resourceManager))
resourcesGroup.PUT("copy", HandleCopy(resourceManager))
resourcesGroup.DELETE("", HandleDelete(resourceManager))
v1Group.POST("", HandleUpload(resourceManager))
v1Group.GET("", HandleDownload(resourceManager))
v1Group.GET("presign", HandlePresign(resourceManager))
v1Group.PUT("copy", HandleCopy(resourceManager))
v1Group.DELETE("", HandleDelete(resourceManager))
server.GET("/api/v1/resources", Secure("basic", "api", "token"), HandleDownload(resourceManager))
}
func loadExpiration() time.Duration {

40
domain/cache/in_mem_cache.go vendored Normal file
View File

@ -0,0 +1,40 @@
package cache
import (
"errors"
"time"
)
func newInMemCache() Manager {
return &inMemCache{make(map[string]inMemEntry)}
}
type inMemEntry struct {
value string
expiration time.Time
}
type inMemCache struct {
entries map[string]inMemEntry
}
func (i *inMemCache) Set(key, value string, expiration time.Duration) error {
i.entries[key] = inMemEntry{
value: value,
expiration: time.Now().Add(expiration),
}
return nil
}
func (i *inMemCache) Get(key string) (string, error) {
if val, exists := i.entries[key]; exists {
if val.expiration.Before(time.Now()) {
delete(i.entries, key)
} else {
return val.value, nil
}
}
return "", errors.New("no value for given key")
}
func (i *inMemCache) Remove(key string) {
delete(i.entries, key)
}

35
domain/cache/impl.go → domain/cache/redis_cache.go vendored
View File

@ -2,7 +2,6 @@ package cache
import (
"context"
"errors"
"github.com/go-redis/redis/v8"
"os"
"time"
@ -37,37 +36,3 @@ func (r *redisCache) Remove(key string) {
defer cancel()
r.client.Del(ctx, key)
}
func newInMemCache() Manager {
return &inMemCache{make(map[string]inMemEntry)}
}
type inMemEntry struct {
value string
expiration time.Time
}
type inMemCache struct {
entries map[string]inMemEntry
}
func (i *inMemCache) Set(key, value string, expiration time.Duration) error {
i.entries[key] = inMemEntry{
value: value,
expiration: time.Now().Add(expiration),
}
return nil
}
func (i *inMemCache) Get(key string) (string, error) {
if val, exists := i.entries[key]; exists {
if val.expiration.Before(time.Now()) {
delete(i.entries, key)
} else {
return val.value, nil
}
}
return "", errors.New("no value for given key")
}
func (i *inMemCache) Remove(key string) {
delete(i.entries, key)
}

113
domain/resource/file_manager.go Normal file
View File

@ -0,0 +1,113 @@
package resource
import (
"context"
"io/ioutil"
"log"
"os"
"path/filepath"
"resource-manager/domain/cache"
"resource-manager/security"
"strings"
"time"
)
func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manager {
path := os.Getenv("LOCAL_PATH_PREFIX")
// if path isn't set use local
if path == "" {
path, _ = os.Getwd()
}
log.Println("Manager | using path (" + path + ") and expiration of " + expiration.String())
manager := fileManager{cacheManager, path, expiration}
return &manager
}
type fileManager struct {
cache cache.Manager
path string
expiration time.Duration
}
func (f *fileManager) Upload(request UploadRequest) {
fullPath := filepath.Join(f.path, request.Path)
createFolder(fullPath)
if checkFileExists(fullPath) && !request.Overwrite {
log.Println("Manager | cannot upload file as file on same path already exists")
return
}
log.Println("Manager | uploading to (" + request.Path + ")")
if err := os.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil {
log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error())
}
}
func checkFileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func (f *fileManager) Download(ctx context.Context, path string) (file []byte, err error) {
fullPath := filepath.Join(f.path, path)
file, err = ioutil.ReadFile(fullPath)
if err != nil {
log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error())
}
return
}
func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
token, err := security.CreateToken(security.PresignToken{Path: path}, f.expiration)
if err != nil {
return "", err
}
return "/api/v1/resources?token=" + token, nil
}
func (f *fileManager) Copy(from string, to string, overwrite bool) error {
fromPath := filepath.Join(f.path, from)
toPath := filepath.Join(f.path, to)
log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")")
createFolder(filepath.Join(toPath, "file"))
return copyFolder(fromPath, toPath, overwrite)
}
func (f *fileManager) Delete(path string) error {
log.Println("Manager | deleting " + path)
fullPath := filepath.Join(f.path, path)
if err := os.Remove(fullPath); err != nil {
log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error())
return err
}
f.cache.Remove(path)
return nil
}
func createFolder(path string) {
paths := strings.Split(path, "/")
folderPath := "/" + filepath.Join(paths[:len(paths)-1]...)
if err := os.MkdirAll(folderPath, 0o755); err != nil {
log.Println("[error] ", err)
}
}
func copyFolder(source, destination string, overwrite bool) error {
var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error {
relPath := strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if d.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
doesExist := false
if _, err := os.Stat(filepath.Join(destination, relPath)); !os.IsNotExist(err) {
doesExist = true
}
if !doesExist || overwrite {
var data, err1 = ioutil.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777)
} else {
return nil
}
}
})
return err
}

1
domain/resource/manager.go
View File

@ -22,6 +22,7 @@ func NewManager(cacheManager cache.Manager, expiration time.Duration) Manager {
log.Println("Manager | using S3 for data storage")
return newS3Bucket(cacheManager, expiration)
} else {
log.Println("Manager | using local file for data storage")
return newLocalFolder(cacheManager, expiration)
}
}

97
domain/resource/impl.go → domain/resource/s3_manager.go
View File

@ -8,11 +8,9 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"io/ioutil"
"log"
"net/url"
"os"
"path/filepath"
"resource-manager/domain/cache"
"strconv"
"strings"
@ -20,101 +18,6 @@ import (
"time"
)
func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manager {
path := os.Getenv("UPLOAD_PATH")
// if path isn't set use local
if path == "" {
path, _ = os.Getwd()
}
log.Println("Manager | using local file system for data storage (" + path + ")")
manager := fileManager{cacheManager, path}
return &manager
}
type fileManager struct {
cache cache.Manager
path string
}
func (f *fileManager) Upload(request UploadRequest) {
fullPath := filepath.Join(f.path, request.Path)
createFolder(fullPath)
if checkFileExists(fullPath) && !request.Overwrite {
log.Println("Manager | cannot upload file as file on same path already exists")
return
}
log.Println("Manager | uploading to (" + request.Path + ")")
if err := os.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil {
log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error())
}
}
func checkFileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func (f *fileManager) Download(ctx context.Context, path string) (file []byte, err error) {
fullPath := filepath.Join(f.path, path)
file, err = ioutil.ReadFile(fullPath)
if err != nil {
log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error())
}
return
}
func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
return "/api/v1/get?path=" + path, nil
}
func (f *fileManager) Copy(from string, to string, overwrite bool) error {
fromPath := filepath.Join(f.path, from)
toPath := filepath.Join(f.path, to)
log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")")
createFolder(filepath.Join(toPath, "file"))
return copyFolder(fromPath, toPath, overwrite)
}
func (f *fileManager) Delete(path string) error {
log.Println("Manager | deleting " + path)
fullPath := filepath.Join(f.path, path)
if err := os.Remove(fullPath); err != nil {
log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error())
return err
}
f.cache.Remove(path)
return nil
}
func createFolder(path string) {
paths := strings.Split(path, "/")
folderPath := "/" + filepath.Join(paths[:len(paths)-1]...)
if err := os.MkdirAll(folderPath, 0o755); err != nil {
log.Println("[error] ", err)
}
}
func copyFolder(source, destination string, overwrite bool) error {
var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error {
relPath := strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if d.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
doesExist := false
if _, err := os.Stat(filepath.Join(destination, relPath)); !os.IsNotExist(err) {
doesExist = true
}
if !doesExist || overwrite {
var data, err1 = ioutil.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777)
} else {
return nil
}
}
})
return err
}
func newS3Bucket(cacheManager cache.Manager, expiration time.Duration) Manager {
sess, err := session.NewSession(&aws.Config{
Region: aws.String(os.Getenv("AWS_REGION_NAME"))},

1
go.mod
View File

@ -17,6 +17,7 @@ require (
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect

2
go.sum
View File

@ -33,6 +33,8 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=

52
security/generator.go Normal file
View File

@ -0,0 +1,52 @@
package security
import (
"errors"
"github.com/golang-jwt/jwt"
"os"
"time"
)
type PresignToken struct {
Path string
}
func CreateToken(tokenInfo PresignToken, duration time.Duration) (string, error) {
// jwt token
atClaims := jwt.MapClaims{}
atClaims["authorized"] = true
// user info
atClaims["path"] = tokenInfo.Path
// expiration
atClaims["exp"] = time.Now().Add(duration).Unix()
at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims)
secret := getSecret()
return at.SignedString([]byte(secret))
}
func ParseToken(token string) (PresignToken, error) {
secret := getSecret()
parsedToken, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) {
return []byte(secret), nil
})
if err != nil {
return PresignToken{}, err
}
claims, ok := parsedToken.Claims.(jwt.MapClaims)
if ok {
return PresignToken{
Path: claims["path"].(string),
}, nil
} else {
return PresignToken{}, errors.New("failed parsing token")
}
}
func getSecret() string {
secret, exists := os.LookupEnv("JWT_SECRET")
if !exists {
panic("env variable JWT_SECRET not set")
}
return secret
}