brajkovic
/
resource_manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added logging

This commit is contained in:
Borna Rajkovic 2021-11-18 19:52:40 +01:00
parent 4929147587
commit 7883b71c76
4 changed files with 51 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cache/manager.go vendored
View File

@ -1,6 +1,7 @@
package cache
import (
"log"
"os"
"strings"
"time"
@ -8,8 +9,10 @@ import (
func NewManager() Manager {
if strings.Contains(os.Getenv("PROFILE"), "redis") {
log.Println("Cache | using redis cache")
return newRedisCache()
} else {
log.Println("Cache | using in memory cache")
return newInMemCache()
}
}
@ -19,4 +22,3 @@ type Manager interface {
Get(key string) (string, error)
Remove(key string)
}

18
main.go
View File

@ -13,13 +13,16 @@ import (
)
func init() {
log.SetPrefix("[ReM]")
log.SetPrefix("[ReM] ")
godotenv.Load()
}
func main() {
cacheManager := cache.NewManager()
resourceManager := resource.NewManager(cacheManager, 1 * time.Hour)
expiration := loadExpiration()
log.Println("Presign | expiration set to " + expiration.String())
resourceManager := resource.NewManager(cacheManager, expiration)
server := gin.Default()
if strings.Contains(os.Getenv("PROFILE"), "legacy") {
@ -29,6 +32,17 @@ func main() {
log.Fatalln(http.ListenAndServe(":5201", server))
}
func loadExpiration() time.Duration {
if value := os.Getenv("PRESIGN_DURATION"); value != "" {
duration, err := time.ParseDuration(value)
if err != nil {
return duration
}
}
// default duration
return 1 * time.Hour
}
func setupLegacyEndpoints(server *gin.Engine, resourceManager resource.Manager) {
server.POST("/save", HandleLegacySave(resourceManager))
server.GET("/get", HandleGet(resourceManager))

38
resource/impl.go
View File

@ -14,6 +14,7 @@ import (
"os"
"path/filepath"
"resource_manager/cache"
"strconv"
"strings"
"sync"
"time"
@ -25,6 +26,7 @@ func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manage
if path == "" {
path, _ = os.Getwd()
}
log.Println("Manager | using local file system for data storage (" + path + ")")
manager := fileManager{cacheManager, path}
return &manager
}
@ -37,13 +39,17 @@ type fileManager struct {
func (f *fileManager) Upload(request UploadRequest) {
fullPath := filepath.Join(f.path, request.Path)
createFolder(fullPath)
log.Println("Manager | uploading to (" + request.Path + ")")
if err := ioutil.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil {
log.Println("[error] ", err)
log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error())
}
}
func (f *fileManager) Download(ctx context.Context, path string) (file []byte, err error) {
fullPath := filepath.Join(f.path, path)
file, err = ioutil.ReadFile(fullPath)
if err != nil {
log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error())
}
return
}
func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
@ -52,18 +58,20 @@ func (f *fileManager) Presign(_ context.Context, path string) (string, error) {
func (f *fileManager) Copy(from string, to string, overwrite bool) error {
fromPath := filepath.Join(f.path, from)
toPath := filepath.Join(f.path, to)
log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")")
createFolder(filepath.Join(toPath, "file"))
return copyFolder(fromPath, toPath, overwrite)
}
func (f *fileManager) Delete(path string) error {
log.Println("Manager | deleting " + path)
fullPath := filepath.Join(f.path, path)
if err := os.Remove(fullPath); err != nil {
log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error())
return err
}
f.cache.Remove(path)
return nil
}
func createFolder(path string) {
paths := strings.Split(path, "/")
folderPath := "/" + filepath.Join(paths[:len(paths)-1]...)
@ -73,7 +81,6 @@ func createFolder(path string) {
}
func copyFolder(source, destination string, overwrite bool) error {
var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error {
relPath := strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
@ -169,18 +176,18 @@ func (a *awsManager) Copy(from string, to string, overwrite bool) error {
fromPath := strings.SplitN(from, "/", 2)
toPath := strings.SplitN(to, "/", 2)
output, err := a.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: aws.String(fromPath[0]),
Prefix: aws.String(fromPath[1]),
Bucket: aws.String(fromPath[0]),
Prefix: aws.String(fromPath[1]),
})
if err != nil {
return err
}
for _, obj := range output.Contents {
if overwrite || !a.doesFileExist(strings.Replace(*obj.Key, fromPath[1], toPath[1], 1)) {
if overwrite || !a.doesFileExist(strings.Replace(*obj.Key, fromPath[1], toPath[1], 1)) {
_, err = a.downloader.S3.CopyObject(&s3.CopyObjectInput{
Bucket: aws.String(toPath[0]),
CopySource: aws.String(url.PathEscape(from)),
Key: aws.String(toPath[1]),
Bucket: aws.String(toPath[0]),
CopySource: aws.String(url.PathEscape(from)),
Key: aws.String(toPath[1]),
})
if err != nil {
return err
@ -193,7 +200,7 @@ func (a *awsManager) Delete(path string) error {
paths := strings.SplitN(path, "/", 2)
_, err := a.uploader.S3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(paths[0]),
Key: aws.String(paths[1]),
Key: aws.String(paths[1]),
})
if err != nil {
return err
@ -204,6 +211,7 @@ func (a *awsManager) Delete(path string) error {
func (a *awsManager) addToCurrentlyUploading(path string) {
a.uploadSync.Lock()
log.Println("Manager | (" + path + ") added to upload list")
a.currentlyUploading[path] = make([]onUpload, 0)
a.uploadSync.Unlock()
}
@ -221,6 +229,8 @@ func (a *awsManager) notifyOnUploadComplete(path string, status bool) {
a.uploadSync.Lock()
defer a.uploadSync.Unlock()
log.Println("Manager | (" + path + ") finished uploading, success: " + strconv.FormatBool(status))
for _, upload := range a.currentlyUploading[path] {
upload <- status
close(upload)
@ -229,6 +239,7 @@ func (a *awsManager) notifyOnUploadComplete(path string, status bool) {
}
func (a *awsManager) upload(path string, data []byte, mimeType string) bool {
log.Println("Manager | (" + path + ") uploading")
paths := strings.SplitN(path, "/", 2)
_, err := a.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(paths[0]),
@ -245,6 +256,9 @@ func (a *awsManager) presign(path string, expiration time.Duration) (string, err
Key: aws.String(paths[1]),
}
request, _ := a.downloader.S3.GetObjectRequest(&requestInput)
if request.Error != nil {
log.Println("Manager | (" + path + ") failed presiging, cause: " + request.Error.Error())
}
return request.Presign(expiration)
}
func (a *awsManager) download(path string) ([]byte, error) {
@ -255,6 +269,9 @@ func (a *awsManager) download(path string) ([]byte, error) {
}
buf := aws.NewWriteAtBuffer([]byte{})
_, err := a.downloader.Download(buf, &requestInput)
if err != nil {
log.Println("Manager | (" + path + ") failed downloading, cause: " + err.Error())
}
return buf.Bytes(), err
}
func (a *awsManager) doesFileExist(path string) bool {
@ -269,4 +286,3 @@ func (a *awsManager) doesFileExist(path string) bool {
}
return false
}

8
resource/manager.go
View File

@ -3,6 +3,7 @@ package resource
import (
"bytes"
"context"
"log"
"os"
"resource_manager/cache"
"strings"
@ -10,14 +11,15 @@ import (
)
type UploadRequest struct {
Buffer *bytes.Buffer
Path string
MimeType string
Buffer *bytes.Buffer
Path string
MimeType string
Overwrite bool
}
func NewManager(cacheManager cache.Manager, expiration time.Duration) Manager {
if strings.Contains(os.Getenv("PROFILE"), "s3") {
log.Println("Manager | using S3 for data storage")
return newS3Bucket(cacheManager, expiration)
} else {
return newLocalFolder(cacheManager, expiration)