diff --git a/cache/manager.go b/cache/manager.go index d9677f6..18d85a8 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -1,6 +1,7 @@ package cache import ( + "log" "os" "strings" "time" @@ -8,8 +9,10 @@ import ( func NewManager() Manager { if strings.Contains(os.Getenv("PROFILE"), "redis") { + log.Println("Cache | using redis cache") return newRedisCache() } else { + log.Println("Cache | using in memory cache") return newInMemCache() } } @@ -19,4 +22,3 @@ type Manager interface { Get(key string) (string, error) Remove(key string) } - diff --git a/main.go b/main.go index b55fcf3..603a6e8 100644 --- a/main.go +++ b/main.go @@ -13,13 +13,16 @@ import ( ) func init() { - log.SetPrefix("[ReM]") + log.SetPrefix("[ReM] ") godotenv.Load() } func main() { cacheManager := cache.NewManager() - resourceManager := resource.NewManager(cacheManager, 1 * time.Hour) + expiration := loadExpiration() + log.Println("Presign | expiration set to " + expiration.String()) + + resourceManager := resource.NewManager(cacheManager, expiration) server := gin.Default() if strings.Contains(os.Getenv("PROFILE"), "legacy") { @@ -29,6 +32,17 @@ func main() { log.Fatalln(http.ListenAndServe(":5201", server)) } +func loadExpiration() time.Duration { + if value := os.Getenv("PRESIGN_DURATION"); value != "" { + duration, err := time.ParseDuration(value) + if err != nil { + return duration + } + } + // default duration + return 1 * time.Hour +} + func setupLegacyEndpoints(server *gin.Engine, resourceManager resource.Manager) { server.POST("/save", HandleLegacySave(resourceManager)) server.GET("/get", HandleGet(resourceManager)) diff --git a/resource/impl.go b/resource/impl.go index 8ecc10b..8a22250 100644 --- a/resource/impl.go +++ b/resource/impl.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "resource_manager/cache" + "strconv" "strings" "sync" "time" @@ -25,6 +26,7 @@ func newLocalFolder(cacheManager cache.Manager, expiration time.Duration) Manage if path == "" { path, _ = os.Getwd() } + log.Println("Manager | using local file system for data storage (" + path + ")") manager := fileManager{cacheManager, path} return &manager } @@ -37,13 +39,17 @@ type fileManager struct { func (f *fileManager) Upload(request UploadRequest) { fullPath := filepath.Join(f.path, request.Path) createFolder(fullPath) + log.Println("Manager | uploading to (" + request.Path + ")") if err := ioutil.WriteFile(fullPath, request.Buffer.Bytes(), 0o644); err != nil { - log.Println("[error] ", err) + log.Println("Manager | failed uploading (" + request.Path + ") cause: " + err.Error()) } } func (f *fileManager) Download(ctx context.Context, path string) (file []byte, err error) { fullPath := filepath.Join(f.path, path) file, err = ioutil.ReadFile(fullPath) + if err != nil { + log.Println("Manager | failed downloading (" + path + ") cause: " + err.Error()) + } return } func (f *fileManager) Presign(_ context.Context, path string) (string, error) { @@ -52,18 +58,20 @@ func (f *fileManager) Presign(_ context.Context, path string) (string, error) { func (f *fileManager) Copy(from string, to string, overwrite bool) error { fromPath := filepath.Join(f.path, from) toPath := filepath.Join(f.path, to) + log.Println("Manager | copying from (" + fromPath + ") to (" + toPath + ")") createFolder(filepath.Join(toPath, "file")) return copyFolder(fromPath, toPath, overwrite) } func (f *fileManager) Delete(path string) error { + log.Println("Manager | deleting " + path) fullPath := filepath.Join(f.path, path) if err := os.Remove(fullPath); err != nil { + log.Println("Manager | failed deleting (" + path + ") cause: " + err.Error()) return err } f.cache.Remove(path) return nil } - func createFolder(path string) { paths := strings.Split(path, "/") folderPath := "/" + filepath.Join(paths[:len(paths)-1]...) @@ -73,7 +81,6 @@ func createFolder(path string) { } func copyFolder(source, destination string, overwrite bool) error { var err = filepath.WalkDir(source, func(path string, d os.DirEntry, err error) error { - relPath := strings.Replace(path, source, "", 1) if relPath == "" { return nil @@ -169,18 +176,18 @@ func (a *awsManager) Copy(from string, to string, overwrite bool) error { fromPath := strings.SplitN(from, "/", 2) toPath := strings.SplitN(to, "/", 2) output, err := a.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(fromPath[0]), - Prefix: aws.String(fromPath[1]), + Bucket: aws.String(fromPath[0]), + Prefix: aws.String(fromPath[1]), }) if err != nil { return err } for _, obj := range output.Contents { - if overwrite || !a.doesFileExist(strings.Replace(*obj.Key, fromPath[1], toPath[1], 1)) { + if overwrite || !a.doesFileExist(strings.Replace(*obj.Key, fromPath[1], toPath[1], 1)) { _, err = a.downloader.S3.CopyObject(&s3.CopyObjectInput{ - Bucket: aws.String(toPath[0]), - CopySource: aws.String(url.PathEscape(from)), - Key: aws.String(toPath[1]), + Bucket: aws.String(toPath[0]), + CopySource: aws.String(url.PathEscape(from)), + Key: aws.String(toPath[1]), }) if err != nil { return err @@ -193,7 +200,7 @@ func (a *awsManager) Delete(path string) error { paths := strings.SplitN(path, "/", 2) _, err := a.uploader.S3.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(paths[0]), - Key: aws.String(paths[1]), + Key: aws.String(paths[1]), }) if err != nil { return err @@ -204,6 +211,7 @@ func (a *awsManager) Delete(path string) error { func (a *awsManager) addToCurrentlyUploading(path string) { a.uploadSync.Lock() + log.Println("Manager | (" + path + ") added to upload list") a.currentlyUploading[path] = make([]onUpload, 0) a.uploadSync.Unlock() } @@ -221,6 +229,8 @@ func (a *awsManager) notifyOnUploadComplete(path string, status bool) { a.uploadSync.Lock() defer a.uploadSync.Unlock() + log.Println("Manager | (" + path + ") finished uploading, success: " + strconv.FormatBool(status)) + for _, upload := range a.currentlyUploading[path] { upload <- status close(upload) @@ -229,6 +239,7 @@ func (a *awsManager) notifyOnUploadComplete(path string, status bool) { } func (a *awsManager) upload(path string, data []byte, mimeType string) bool { + log.Println("Manager | (" + path + ") uploading") paths := strings.SplitN(path, "/", 2) _, err := a.uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(paths[0]), @@ -245,6 +256,9 @@ func (a *awsManager) presign(path string, expiration time.Duration) (string, err Key: aws.String(paths[1]), } request, _ := a.downloader.S3.GetObjectRequest(&requestInput) + if request.Error != nil { + log.Println("Manager | (" + path + ") failed presiging, cause: " + request.Error.Error()) + } return request.Presign(expiration) } func (a *awsManager) download(path string) ([]byte, error) { @@ -255,6 +269,9 @@ func (a *awsManager) download(path string) ([]byte, error) { } buf := aws.NewWriteAtBuffer([]byte{}) _, err := a.downloader.Download(buf, &requestInput) + if err != nil { + log.Println("Manager | (" + path + ") failed downloading, cause: " + err.Error()) + } return buf.Bytes(), err } func (a *awsManager) doesFileExist(path string) bool { @@ -269,4 +286,3 @@ func (a *awsManager) doesFileExist(path string) bool { } return false } - diff --git a/resource/manager.go b/resource/manager.go index 94795a7..a50f512 100644 --- a/resource/manager.go +++ b/resource/manager.go @@ -3,6 +3,7 @@ package resource import ( "bytes" "context" + "log" "os" "resource_manager/cache" "strings" @@ -10,14 +11,15 @@ import ( ) type UploadRequest struct { - Buffer *bytes.Buffer - Path string - MimeType string + Buffer *bytes.Buffer + Path string + MimeType string Overwrite bool } func NewManager(cacheManager cache.Manager, expiration time.Duration) Manager { if strings.Contains(os.Getenv("PROFILE"), "s3") { + log.Println("Manager | using S3 for data storage") return newS3Bucket(cacheManager, expiration) } else { return newLocalFolder(cacheManager, expiration)