From 019caf3bf54cf21ad9ba4d1067d1a85b18692592 Mon Sep 17 00:00:00 2001 From: yangpeng Date: Tue, 8 Aug 2023 13:47:21 +0800 Subject: [PATCH] add last modify time and size filter --- lib/command.go | 53 +++++ lib/command_test.go | 205 +++++++++++++++++ lib/const.go | 2 + lib/cp.go | 96 +++++++- lib/cp_test.go | 547 ++++++++++++++++++++++++++++++++++++++++++++ lib/option.go | 6 + lib/sync.go | 242 ++++++++++++++++++-- lib/sync_test.go | 156 +++++++++++++ lib/util.go | 227 +++++------------- 9 files changed, 1342 insertions(+), 192 deletions(-) diff --git a/lib/command.go b/lib/command.go index cdbe7704..a0540b50 100644 --- a/lib/command.go +++ b/lib/command.go @@ -2,6 +2,7 @@ package lib import ( "fmt" + "io/fs" "net" "net/http" "net/url" @@ -52,6 +53,8 @@ type Command struct { options OptionMapType configOptions OptionMapType inputKeySecret string + isFilter bool + bFilter bool } // Commander is the interface of all commands @@ -840,6 +843,56 @@ func (cmd *Command) getOSSTagging(strTagging string) ([]oss.Tag, error) { return tags, nil } +func (cmd *Command) checkFilter() error { + startTime, _ := GetInt(OptionStartTime, cmd.options) + endTime, _ := GetInt(OptionEndTime, cmd.options) + maxSize, _ := GetInt(OptionMaxSize, cmd.options) + minSize, _ := GetInt(OptionMinSize, cmd.options) + + if startTime != 0 || endTime != 0 || maxSize != 0 || minSize != 0 { + cmd.isFilter = true + } + if endTime > 0 && startTime > endTime { + return fmt.Errorf("--start-time %d is larger than --end-time %d", startTime, endTime) + } + + if maxSize > 0 && minSize > maxSize { + return fmt.Errorf("--min-size %d is larger than --max-size %d", minSize, maxSize) + } + return nil +} + +func (cmd *Command) filterLocalFile(fileInfo fs.FileInfo) bool { + filterMap := cmd.getFilterMap() + return CheckLocalFile(filterMap, fileInfo) +} + +func (cmd *Command) filterObject(object oss.ObjectProperties) bool { + filterMap := cmd.getFilterMap() + return CheckObject(filterMap, object) +} + +func (cmd *Command) getFilterMap() map[string]int64 { + startTime, _ := GetInt(OptionStartTime, cmd.options) + endTime, _ := GetInt(OptionEndTime, cmd.options) + maxSize, _ := GetInt(OptionMaxSize, cmd.options) + minSize, _ := GetInt(OptionMinSize, cmd.options) + filterMap := map[string]int64{} + if endTime != 0 { + filterMap[OptionEndTime] = endTime + } + if startTime != 0 { + filterMap[OptionStartTime] = startTime + } + if maxSize != 0 { + filterMap[OptionMaxSize] = maxSize + } + if minSize != 0 { + filterMap[OptionMinSize] = minSize + } + return filterMap +} + // GetAllCommands returns all commands list func GetAllCommands() []interface{} { return []interface{}{ diff --git a/lib/command_test.go b/lib/command_test.go index 7fe999f6..def96a79 100644 --- a/lib/command_test.go +++ b/lib/command_test.go @@ -2928,3 +2928,208 @@ func (s *OssutilCommandSuite) TestCommandObjectProducer(c *C) { c.Assert(true, Equals, false) } } + +//Test FilterLocalFile +func (s *OssutilCommandSuite) TestFilterLocalFileWithSizeAndTime(c *C) { + str := int64(0) + minSize := int64(10) + maxSize := int64(20) + maxTime := strconv.Itoa(int(time.Now().Unix())) + minTime := strconv.Itoa(int(time.Now().Add(-90 * time.Second).Unix())) + minSize1 := int64(2) + f, _ := os.Stat(uploadFileName) + testLogger.Println(f.Size()) + testLogger.Println(f.ModTime()) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize, + } + + next := copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize1, + } + + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionMaxSize: &minSize1, + OptionMinSize: &str, + } + + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMaxSize: &minSize, + OptionMinSize: &str, + } + + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize, + OptionMaxSize: &maxSize, + } + + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize1, + OptionMaxSize: &maxSize, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionStartTime: &minTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionStartTime: &maxTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionMinSize: &str, + OptionMaxSize: &str, + OptionStartTime: &str, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionStartTime: &minTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionStartTime: &minTime, + OptionMinSize: &minSize1, + OptionMaxSize: &maxSize, + } + next = copyCommand.command.filterLocalFile(f) + c.Assert(next, Equals, true) +} + +//Test FilterObject +func (s *OssutilCommandSuite) TestFilterObjectWithSizeAndTime(c *C) { + str := int64(0) + minSize := int64(10) + maxSize := int64(20) + maxTime := strconv.Itoa(int(time.Now().Unix())) + minTime := strconv.Itoa(int(time.Now().Add(-90 * time.Second).Unix())) + minSize1 := int64(2) + + var f oss.ObjectProperties + f.Size = 3 + f.LastModified = time.Now().Add(-50 * time.Second) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize, + } + + next := copyCommand.command.filterObject(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize1, + } + + testLogger.Println(copyCommand.command.options) + + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionMaxSize: &minSize1, + OptionMinSize: &str, + } + + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMaxSize: &minSize, + OptionMinSize: &str, + } + + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize, + OptionMaxSize: &maxSize, + } + + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionMinSize: &minSize1, + OptionMaxSize: &maxSize, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionStartTime: &minTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionStartTime: &maxTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, false) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionMinSize: &str, + OptionMaxSize: &str, + OptionStartTime: &str, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionStartTime: &minTime, + OptionMinSize: &str, + OptionMaxSize: &str, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) + + copyCommand.command.options = OptionMapType{ + OptionEndTime: &maxTime, + OptionStartTime: &minTime, + OptionMinSize: &minSize1, + OptionMaxSize: &maxSize, + } + next = copyCommand.command.filterObject(f) + c.Assert(next, Equals, true) +} diff --git a/lib/const.go b/lib/const.go index 1e25e3bc..62f21c1c 100644 --- a/lib/const.go +++ b/lib/const.go @@ -104,6 +104,8 @@ const ( OptionRegion = "region" OptionCloudBoxID = "cloudBoxID" OptionQueryParam = "queryParam" + OptionMaxSize = "maxSize" + OptionMinSize = "minSize" ) // the elements show in stat object diff --git a/lib/cp.go b/lib/cp.go index a4e39c0b..16ac6008 100644 --- a/lib/cp.go +++ b/lib/cp.go @@ -1294,6 +1294,10 @@ var copyCommand = CopyCommand{ OptionSignVersion, OptionRegion, OptionCloudBoxID, + OptionStartTime, + OptionEndTime, + OptionMaxSize, + OptionMinSize, }, }, } @@ -1473,6 +1477,11 @@ func (cc *CopyCommand) RunCommand() error { cc.cpOption.partitionCount = 0 } + err = cc.command.checkFilter() + if err != nil { + return err + } + cc.monitor.init(opType) cc.cpOption.opType = opType @@ -1707,6 +1716,10 @@ func (cc *CopyCommand) fileStatistic(srcURLList []StorageURLer) { return } } else { + next := cc.command.filterLocalFile(f) + if !next { + continue + } if cc.filterPath(name, cc.cpOption.cpDir) { cc.monitor.updateScanSizeNum(f.Size(), 1) } @@ -1734,6 +1747,10 @@ func (cc *CopyCommand) getCurrentDirFilesStatistic(dpath string) error { // for symlink continue } + next := cc.command.filterLocalFile(realInfo) + if !next { + continue + } if doesSingleFileMatchPatterns(fileInfo.Name(), cc.cpOption.filters) { cc.monitor.updateScanSizeNum(fileInfo.Size(), 1) @@ -1758,7 +1775,12 @@ func (cc *CopyCommand) getFileListStatistic(dpath string) error { if !cc.filterPath(fpath, cc.cpOption.cpDir) { return nil } - + if !f.IsDir() { + next := cc.command.filterLocalFile(f) + if !next { + return nil + } + } realFileSize := f.Size() dpath = filepath.Clean(dpath) fpath = filepath.Clean(fpath) @@ -1835,6 +1857,7 @@ func (cc *CopyCommand) fileProducer(srcURLList []StorageURLer, chFiles chan<- fi chListError <- err return } + if f.IsDir() { if !strings.HasSuffix(name, string(os.PathSeparator)) { // for link directory @@ -1847,6 +1870,10 @@ func (cc *CopyCommand) fileProducer(srcURLList []StorageURLer, chFiles chan<- fi return } } else { + next := cc.command.filterLocalFile(f) + if !next { + continue + } dir, fname := filepath.Split(name) chFiles <- fileInfoType{fname, dir} } @@ -1872,6 +1899,11 @@ func (cc *CopyCommand) getCurrentDirFileList(dpath string, chFiles chan<- fileIn continue } + next := cc.command.filterLocalFile(realInfo) + if !next { + continue + } + if doesSingleFileMatchPatterns(fileInfo.Name(), cc.cpOption.filters) { chFiles <- fileInfoType{fileInfo.Name(), dpath} } @@ -1909,6 +1941,11 @@ func (cc *CopyCommand) getFileList(dpath string, chFiles chan<- fileInfoType) er } } return nil + } else { + next := cc.command.filterLocalFile(f) + if !next { + return nil + } } if cc.cpOption.disableAllSymlink && (f.Mode()&os.ModeSymlink) != 0 { @@ -1931,6 +1968,11 @@ func (cc *CopyCommand) getFileList(dpath string, chFiles chan<- fileInfoType) er linkDir := name + fileName + string(os.PathSeparator) symlinkDiretorys = append(symlinkDiretorys, linkDir) return nil + }else{ + next := cc.command.filterLocalFile(realInfo) + if !next { + return nil + } } } @@ -2368,7 +2410,6 @@ func (cc *CopyCommand) downloadFiles(srcURL CloudURL, destURL FileURL) error { prefix = srcURL.object[:index+1] relativeKey = srcURL.object[index+1:] } - go cc.objectStatistic(bucket, srcURL) err := cc.downloadSingleFileWithReport(bucket, objectInfoType{prefix, relativeKey, -1, time.Now()}, filePath) return cc.formatResultPrompt(err) @@ -2444,8 +2485,13 @@ func (cc *CopyCommand) downloadSingleFileWithReport(bucket *oss.Bucket, objectIn LogInfo("download success,object:%s,size:%d,speed:%.2f(KB/s),cost:%d(ms)\n", objectKey, realSize, speed, cost) cc.updateSnapshot(nil, CloudURLToString(bucket.BucketName, objectKey), objectInfo.lastModified.Unix()) } - - cc.updateMonitor(skip, err, false, size) + if cc.command.isFilter { + if !cc.command.bFilter { + cc.updateMonitor(skip, err, false, size) + } + } else { + cc.updateMonitor(skip, err, false, size) + } cc.report(msg, err) return err } @@ -2475,6 +2521,14 @@ func (cc *CopyCommand) downloadSingleFile(bucket *oss.Bucket, objectInfo objectI if srct, err = time.Parse(http.TimeFormat, props.Get(oss.HTTPHeaderLastModified)); err != nil { return false, err, size, msg } + var ossObjcet oss.ObjectProperties + ossObjcet.LastModified = srct + ossObjcet.Size = size + next := cc.command.filterObject(ossObjcet) + if !next { + cc.command.bFilter = true + return false, nil, size, msg + } } rsize := cc.getRangeSize(size) @@ -2671,6 +2725,10 @@ func (cc *CopyCommand) objectStatistic(bucket *oss.Bucket, cloudURL CloudURL) { } for _, object := range lor.Objects { + next := cc.command.filterObject(object) + if !next { + continue + } if doesSingleObjectMatchPatterns(object.Key, cc.cpOption.filters) { if cc.cpOption.partitionIndex == 0 || (cc.cpOption.partitionIndex > 0 && matchHash(fnvIns, object.Key, cc.cpOption.partitionIndex-1, cc.cpOption.partitionCount)) { if strings.ToLower(object.Type) == "symlink" && cc.cpOption.opType == operationTypeGet { @@ -2708,9 +2766,16 @@ func (cc *CopyCommand) objectStatistic(bucket *oss.Bucket, cloudURL CloudURL) { cc.monitor.setScanError(err) return } + var object oss.ObjectProperties + object.Size = size + LastModified, _ := time.Parse(http.TimeFormat, props.Get(oss.HTTPHeaderLastModified)) + object.LastModified = LastModified + next := cc.command.filterObject(object) + if !next { + return + } cc.monitor.updateScanSizeNum(cc.getRangeSize(size), 1) } - cc.monitor.setScanEnd() freshProgress() } @@ -2787,6 +2852,10 @@ func (cc *CopyCommand) objectProducer(bucket *oss.Bucket, cloudURL CloudURL, chO return } for _, object := range lor.Objects { + next := cc.command.filterObject(object) + if !next { + continue + } prefix := "" relativeKey := object.Key index := strings.LastIndex(cloudURL.object, "/") @@ -2927,7 +2996,13 @@ func (cc *CopyCommand) checkCopyFileArgs(srcURL, destURL CloudURL) error { func (cc *CopyCommand) copySingleFileWithReport(bucket *oss.Bucket, objectInfo objectInfoType, srcURL, destURL CloudURL) error { skip, err, size, msg := cc.copySingleFile(bucket, objectInfo, srcURL, destURL) - cc.updateMonitor(skip, err, false, size) + if cc.command.isFilter { + if !cc.command.bFilter { + cc.updateMonitor(skip, err, false, size) + } + } else { + cc.updateMonitor(skip, err, false, size) + } cc.report(msg, err) return err } @@ -2959,6 +3034,15 @@ func (cc *CopyCommand) copySingleFile(bucket *oss.Bucket, objectInfo objectInfoT if srct, err = time.Parse(http.TimeFormat, props.Get(oss.HTTPHeaderLastModified)); err != nil { return false, err, size, msg } + + var ossObjcet oss.ObjectProperties + ossObjcet.LastModified = srct + ossObjcet.Size = size + next := cc.command.filterObject(ossObjcet) + if !next { + cc.command.bFilter = true + return false, nil, size, msg + } } if skip, err := cc.skipCopy(destURL, destObject, srct); err != nil || skip { diff --git a/lib/cp_test.go b/lib/cp_test.go index 5034f8b3..e1051168 100644 --- a/lib/cp_test.go +++ b/lib/cp_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "runtime" "strconv" "strings" @@ -6120,3 +6121,549 @@ func (s *OssutilCommandSuite) TestCloudBoxCreateAndDeleteBucket(c *C) { os.Remove(fileName) s.removeBucket(bucketName, true, c) } + +func (s *OssutilCommandSuite) TestCpCmdWithFilterSingleObject(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + objectContext := randLowStr(10) + fileName := "ossutil_test." + randLowStr(12) + s.createFile(fileName, objectContext, c) + + object := randLowStr(12) + cpArgs := []string{fileName, CloudURLToString(bucketName, object)} + + minSize := int64(5) + maxSize := int64(20) + cpDir := CheckpointDir + maxTime := time.Now().Add(+10 * time.Second).Unix() + c.Log(maxTime) + minTime := time.Now().Add(-10 * time.Second).Unix() + //minSize1 := int64(2) + endTime := time.Now().Add(+20 * time.Second).Unix() + str := "" + force := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + OptionForce: &force, + "routines": &routines, + OptionMinSize: &minSize, + } + + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + downFileName := fileName + "-down" + dwArgs := []string{CloudURLToString(bucketName, object), downFileName} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err := ioutil.ReadFile(downFileName) + c.Assert(err, IsNil) + c.Assert(objectContext, Equals, string(fileBody)) + + object2 := object + "2" + cpArgs = []string{fileName, CloudURLToString(bucketName, object2)} + options[OptionMaxSize] = &maxSize + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + downFileName2 := fileName + "-down2" + dwArgs = []string{CloudURLToString(bucketName, object2), downFileName2} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err = ioutil.ReadFile(downFileName2) + c.Assert(err, IsNil) + c.Assert(objectContext, Equals, string(fileBody)) + + object3 := object + "3" + cpArgs = []string{fileName, CloudURLToString(bucketName, object3)} + delete(options, OptionMaxSize) + delete(options, OptionMinSize) + options[OptionStartTime] = &minTime + options[OptionEndTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + time.Sleep(2 * time.Second) + + downFileName3 := fileName + "-down3" + dwArgs = []string{CloudURLToString(bucketName, object3), downFileName3} + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err = ioutil.ReadFile(downFileName3) + c.Assert(err, IsNil) + c.Assert(objectContext, Equals, string(fileBody)) + c.Log(string(fileBody)) + + time.Sleep(2 * time.Second) + + delete(options, OptionStartTime) + delete(options, OptionEndTime) + options[OptionStartTime] = &minTime + downFileName4 := fileName + "-down4" + dwArgs = []string{CloudURLToString(bucketName, object3), downFileName4} + c.Log(dwArgs) + c.Log(options) + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + time.Sleep(2 * time.Second) + + fileBody, err = ioutil.ReadFile(downFileName4) + c.Assert(err, IsNil) + c.Assert(objectContext, Equals, string(fileBody)) + + delete(options, OptionStartTime) + delete(options, OptionEndTime) + options[OptionStartTime] = &maxTime + downFileName5 := fileName + "-down5" + dwArgs = []string{CloudURLToString(bucketName, object3), downFileName5} + c.Log(dwArgs) + c.Log(options) + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err = ioutil.ReadFile(downFileName5) + c.Log(string(fileBody)) + c.Assert(err, NotNil) + + delete(options, OptionStartTime) + delete(options, OptionEndTime) + options[OptionEndTime] = &minTime + downFileName6 := fileName + "-down6" + dwArgs = []string{CloudURLToString(bucketName, object3), downFileName6} + c.Log(dwArgs) + c.Log(options) + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err = ioutil.ReadFile(downFileName6) + c.Log(string(fileBody)) + c.Assert(err, NotNil) + + delete(options, OptionStartTime) + delete(options, OptionEndTime) + options[OptionStartTime] = &maxTime + options[OptionEndTime] = &endTime + downFileName7 := fileName + "-down7" + dwArgs = []string{CloudURLToString(bucketName, object3), downFileName7} + c.Log(dwArgs) + c.Log(options) + _, err = cm.RunCommand("cp", dwArgs, options) + c.Assert(err, IsNil) + + fileBody, err = ioutil.ReadFile(downFileName6) + c.Log(string(fileBody)) + c.Assert(err, NotNil) + + // cp bucket to other bucket + optionsCopy := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + OptionForce: &force, + "routines": &routines, + OptionMinSize: &minSize, + } + cpArgs = []string{CloudURLToString(bucketName, object), CloudURLToString(bucketNameExist, object)} + _, err = cm.RunCommand("cp", cpArgs, optionsCopy) + c.Assert(err, IsNil) + + downCopyName := fileName + "-copy" + s.getObject(bucketNameExist, object, downCopyName, c) + contentCopy := s.readFile(downCopyName, c) + c.Assert(objectContext, Equals, contentCopy) + + cpArgs = []string{CloudURLToString(bucketName, object), CloudURLToString(bucketNameExist, object2)} + optionsCopy[OptionMaxSize] = &maxSize + _, err = cm.RunCommand("cp", cpArgs, optionsCopy) + c.Assert(err, IsNil) + + downCopyName2 := fileName + "-copy2" + s.getObject(bucketNameExist, object2, downCopyName2, c) + contentCopy2 := s.readFile(downCopyName, c) + c.Assert(objectContext, Equals, contentCopy2) + + cpArgs = []string{CloudURLToString(bucketName, object), CloudURLToString(bucketNameExist, object3)} + optionsCopy[OptionStartTime] = &minTime + optionsCopy[OptionEndTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, optionsCopy) + c.Assert(err, IsNil) + + downCopyName3 := fileName + "-copy3" + s.getObject(bucketNameExist, object3, downCopyName3, c) + contentCopy3 := s.readFile(downCopyName, c) + c.Assert(objectContext, Equals, contentCopy3) + object4 := object + "4" + cpArgs = []string{CloudURLToString(bucketName, object), CloudURLToString(bucketNameExist, object4)} + delete(optionsCopy, OptionMaxSize) + delete(optionsCopy, OptionMinSize) + optionsCopy[OptionStartTime] = &minTime + optionsCopy[OptionEndTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, optionsCopy) + c.Assert(err, IsNil) + + downCopyName4 := fileName + "-copy4" + s.getObject(bucketNameExist, object4, downCopyName4, c) + contentCopy4 := s.readFile(downCopyName, c) + c.Assert(objectContext, Equals, contentCopy4) + + s.clearObjects(bucketName, "", c) + os.Remove(downFileName) + os.Remove(fileName) + os.Remove(downFileName2) + os.Remove(downFileName3) + os.Remove(downFileName4) + os.Remove(downCopyName) + os.Remove(downCopyName2) + os.Remove(downCopyName3) + os.Remove(downCopyName4) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCpCmdWithFilterDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + //objectContext := randLowStr(10) + dir := "testdir-inc1" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + cpArgs := []string{dir + "/", CloudURLToString(bucketName, dir+"/")} + minSize := int64(31) + maxSize := int64(34) + cpDir := CheckpointDir + maxTime := time.Now().Add(+10 * time.Second).Unix() + c.Log(maxTime) + minTime := time.Now().Add(-10 * time.Second).Unix() + + endTime := time.Now().Add(+20 * time.Second).Unix() + force := true + recursion := true + routines := strconv.Itoa(Routines) + str := "" + optionsDir := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + OptionForce: &force, + "routines": &routines, + OptionMinSize: &minSize, + OptionRecursion: &recursion, + } + + _, err := cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + downDir := "cp-down-dir" + randLowStr(5) + dwArgs := []string{CloudURLToString(bucketName, dir+"/"), downDir} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist := s.listObjects(bucketNameExist, dir, "ls -", c) + s.clearObjects(bucketNameExist, "", c) + var count, count1, count2 int + err = filepath.Walk(downDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + if !(fileInfo.Size() > minSize) { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + + s.clearObjects(bucketName, "", c) + os.RemoveAll(downDir) + + cpArgs = []string{dir + "/", CloudURLToString(bucketName, dir)} + optionsDir[OptionMaxSize] = &maxSize + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist = s.listLimitedMarker(bucketNameExist, "", "ls ", -1, "", "", c) + s.clearObjects(bucketNameExist, "", c) + + downDir2 := downDir + "2" + dwArgs = []string{CloudURLToString(bucketName, dir+"/"), downDir2} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + count = 0 + count1 = 0 + count2 = 0 + err = filepath.Walk(downDir2, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + if fileInfo.Size() < maxSize && fileInfo.Size() > minSize { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + os.RemoveAll(downDir2) + s.clearObjects(bucketName, "", c) + + cpArgs = []string{dir + "/", CloudURLToString(bucketName, dir)} + delete(optionsDir, OptionMaxSize) + delete(optionsDir, OptionMinSize) + optionsDir[OptionStartTime] = &minTime + optionsDir[OptionEndTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist = s.listLimitedMarker(bucketNameExist, dir+"/", "ls ", -1, "", "", c) + s.clearObjects(bucketNameExist, "", c) + testLogger.Println(objectsExist) + downDir3 := downDir + "3" + dwArgs = []string{CloudURLToString(bucketName, dir+"/"), downDir3} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + count = 0 + count1 = 0 + count2 = 0 + err = filepath.Walk(downDir3, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + end := time.Unix(maxTime, 0) + start := time.Unix(minTime, 0) + if fileInfo.ModTime().Before(end) && fileInfo.ModTime().After(start) { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + s.clearObjects(bucketName, "", c) + os.RemoveAll(downDir3) + + cpArgs = []string{dir + "/", CloudURLToString(bucketName, dir)} + delete(optionsDir, OptionStartTime) + delete(optionsDir, OptionEndTime) + optionsDir[OptionEndTime] = &minTime + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist = s.listLimitedMarker(bucketNameExist, "", "ls ", -1, "", "", c) + s.clearObjects(bucketNameExist, "", c) + + downDir4 := downDir + "4" + dwArgs = []string{CloudURLToString(bucketName, dir+"/"), downDir4} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + count = 0 + count1 = 0 + count2 = 0 + err = filepath.Walk(downDir4, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + t := time.Unix(minTime, 0) + if !fileInfo.ModTime().After(t) { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + c.Assert(count, Equals, len(objectsExist)) + s.clearObjects(bucketName, "", c) + os.RemoveAll(downDir4) + + cpArgs = []string{dir + "/", CloudURLToString(bucketName, dir)} + delete(optionsDir, OptionStartTime) + delete(optionsDir, OptionEndTime) + optionsDir[OptionStartTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist = s.listLimitedMarker(bucketNameExist, "", "ls ", -1, "", "", c) + s.clearObjects(bucketNameExist, "", c) + + downDir5 := downDir + "5" + dwArgs = []string{CloudURLToString(bucketName, dir+"/"), downDir5} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + count = 0 + count1 = 0 + count2 = 0 + err = filepath.Walk(downDir5, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + t := time.Unix(maxTime, 0) + if !fileInfo.ModTime().Before(t) { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + os.RemoveAll(downDir5) + s.clearObjects(bucketName, "", c) + + delete(optionsDir, OptionStartTime) + delete(optionsDir, OptionEndTime) + optionsDir[OptionStartTime] = &maxTime + optionsDir[OptionEndTime] = &maxTime + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + cpArgs = []string{CloudURLToString(bucketName, dir+"/"), CloudURLToString(bucketNameExist, dir+"/")} + _, err = cm.RunCommand("cp", cpArgs, optionsDir) + c.Assert(err, IsNil) + + objectsExist = s.listLimitedMarker(bucketNameExist, "", "ls ", -1, "", "", c) + s.clearObjects(bucketNameExist, "", c) + + downDir6 := downDir + "6" + dwArgs = []string{CloudURLToString(bucketName, dir+"/"), downDir6} + _, err = cm.RunCommand("cp", dwArgs, optionsDir) + c.Assert(err, IsNil) + + count = 0 + count1 = 0 + count2 = 0 + err = filepath.Walk(downDir6, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count++ + } + return nil + }) + c.Assert(err, IsNil) + + for _, filename := range filenames { + fileInfo, _ := os.Stat(dir + "/" + filename) + end := time.Unix(endTime, 0) + start := time.Unix(maxTime, 0) + if fileInfo.ModTime().Before(end) && fileInfo.ModTime().After(start) { + count1++ + } + } + for _, object := range objectsExist { + lastChar := object[len(object)-1:] + if lastChar != "/" { + count2++ + } + } + c.Assert(count, Equals, count2) + c.Assert(count, Equals, count1) + os.RemoveAll(downDir5) + s.clearObjects(bucketName, "", c) + + s.removeBucket(bucketName, true, c) +} diff --git a/lib/option.go b/lib/option.go index f49dfdd4..ff71bdd7 100644 --- a/lib/option.go +++ b/lib/option.go @@ -281,6 +281,12 @@ var OptionMap = map[string]Option{ OptionQueryParam: Option{"", "--query-param", "", OptionTypeStrings, "", "", "设置请求的query参数", "Set the query parameters for the request"}, + OptionMaxSize: Option{"", "--max-size", "", OptionTypeInt64, "", "", + "文件的最大值", + "Maximum file size"}, + OptionMinSize: Option{"", "--min-size", "", OptionTypeInt64, "", "", + "文件的最小值", + "Minimum file size"}, } func (T *Option) getHelp(language string) string { diff --git a/lib/sync.go b/lib/sync.go index 959784d8..b8f0891c 100644 --- a/lib/sync.go +++ b/lib/sync.go @@ -5,9 +5,11 @@ import ( "fmt" oss "github.com/aliyun/aliyun-oss-go-sdk/oss" "io" + "io/ioutil" "os" "path/filepath" "sort" + "strconv" "strings" ) @@ -431,6 +433,11 @@ var syncCommand = SyncCommand{ // The following options are only supported by sc command, not supported by cp command OptionDelete, OptionBackupDir, + + OptionStartTime, + OptionEndTime, + OptionMaxSize, + OptionMinSize, }, }, } @@ -543,13 +550,18 @@ func (sc *SyncCommand) RunCommand() error { } opType := sc.getCommandType(srcURL, destURL) + err = sc.command.checkFilter() + if err != nil { + return err + } + // get file list or object key list srcKeys := make(map[string]string) destKeys := make(map[string]string) if srcURL.IsFileURL() { - err = sc.GetLocalFileKeys(srcURL, srcKeys) + err = sc.GetLocalFileKeys(srcURL, srcKeys, true) } else { - err = sc.GetOssKeys(srcURL, srcKeys) + err = sc.GetOssKeys(srcURL, srcKeys, true) } if err != nil { @@ -557,9 +569,9 @@ func (sc *SyncCommand) RunCommand() error { } if destURL.IsFileURL() { - err = sc.GetLocalFileKeys(destURL, destKeys) + err = sc.GetLocalFileKeys(destURL, destKeys, false) } else { - err = sc.GetOssKeys(destURL, destKeys) + err = sc.GetOssKeys(destURL, destKeys, false) } if err != nil { @@ -716,7 +728,7 @@ func (sc *SyncCommand) BatchRmObjects(bucket *oss.Bucket, objects []string, opti for _, objectKey := range delRes.DeletedObjects { errMsg += (" " + objectKey) } - LogError("delete erro %s\n", errMsg) + LogError("delete error %s\n", errMsg) return fmt.Errorf("delete %s error", errMsg) } @@ -737,7 +749,7 @@ func (sc *SyncCommand) getCommandType(srcURL StorageURLer, destURL StorageURLer) return operationTypePut } -func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]string) error { +func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]string, isSrc bool) error { strPath := sUrl.ToString() if !strings.HasSuffix(strPath, string(os.PathSeparator)) { // for symlink dir @@ -747,7 +759,7 @@ func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]strin chFiles := make(chan fileInfoType, ChannelBuf) chFinish := make(chan error, 2) go sc.ReadLocalFileKeys(chFiles, chFinish, keys) - go sc.GetFileList(strPath, chFiles, chFinish) + go sc.GetFileList(strPath, chFiles, chFinish, isSrc) select { case err := <-chFinish: if err != nil { @@ -757,14 +769,139 @@ func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]strin return nil } -func (sc *SyncCommand) GetFileList(strPath string, chFiles chan<- fileInfoType, chFinish chan<- error) { - err := getFileListCommon(strPath, chFiles, sc.syncOption.onlyCurrentDir, - sc.syncOption.disableAllSymlink, sc.syncOption.enableSymlinkDir, sc.syncOption.filters) +func (sc *SyncCommand) GetFileList(strPath string, chFiles chan<- fileInfoType, chFinish chan<- error, isSrc bool) { + err := sc.getFileListCommon(strPath, chFiles, sc.syncOption.onlyCurrentDir, + sc.syncOption.disableAllSymlink, sc.syncOption.enableSymlinkDir, sc.syncOption.filters, isSrc) if err != nil { chFinish <- err } } +func (sc *SyncCommand) getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir bool, disableAllSymlink bool, + enableSymlinkDir bool, filters []filterOptionType, isSrc bool) error { + defer close(chFiles) + if onlyCurrentDir { + return sc.getCurrentDirFileListCommon(dpath, chFiles, filters, isSrc) + } + + name := dpath + symlinkDiretorys := []string{dpath} + walkFunc := func(fpath string, f os.FileInfo, err error) error { + if f == nil { + return err + } + + dpath = filepath.Clean(dpath) + fpath = filepath.Clean(fpath) + + fileName, err := filepath.Rel(dpath, fpath) + if err != nil { + return fmt.Errorf("list file error: %s, info: %s", fpath, err.Error()) + } + + if f.IsDir() { + if fpath != dpath { + if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { + chFiles <- fileInfoType{fileName, name} + } else { + chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} + } + } + return nil + } else { + if isSrc { + next := sc.command.filterLocalFile(f) + if !next { + return nil + } + } + } + + if disableAllSymlink && (f.Mode()&os.ModeSymlink) != 0 { + return nil + } + + if enableSymlinkDir && (f.Mode()&os.ModeSymlink) != 0 { + // there is difference between os.Stat and os.Lstat in filepath.Walk + realInfo, err := os.Stat(fpath) + if err != nil { + return err + } + + if realInfo.IsDir() { + // it's symlink dir + // if linkDir has suffix os.PathSeparator,os.Lstat determine it is a dir + if !strings.HasSuffix(name, string(os.PathSeparator)) { + name += string(os.PathSeparator) + } + linkDir := name + fileName + string(os.PathSeparator) + symlinkDiretorys = append(symlinkDiretorys, linkDir) + return nil + } else { + if isSrc { + next := sc.command.filterLocalFile(realInfo) + if !next { + return nil + } + } + } + } + + if doesSingleFileMatchPatterns(fileName, filters) { + chFiles <- fileInfoType{fileName, name} + } + return nil + } + + var err error + for { + symlinks := symlinkDiretorys + symlinkDiretorys = []string{} + for _, v := range symlinks { + err = filepath.Walk(v, walkFunc) + if err != nil { + return err + } + } + if len(symlinkDiretorys) == 0 { + break + } + } + return err +} + +func (sc *SyncCommand) getCurrentDirFileListCommon(dpath string, chFiles chan<- fileInfoType, filters []filterOptionType, isSrc bool) error { + if !strings.HasSuffix(dpath, string(os.PathSeparator)) { + dpath += string(os.PathSeparator) + } + + fileList, err := ioutil.ReadDir(dpath) + if err != nil { + return err + } + + for _, fileInfo := range fileList { + if !fileInfo.IsDir() { + realInfo, errF := os.Stat(dpath + fileInfo.Name()) + if errF == nil && realInfo.IsDir() { + // for symlink + continue + } + + if isSrc { + next := sc.command.filterLocalFile(realInfo) + if !next { + continue + } + } + + if doesSingleFileMatchPatterns(fileInfo.Name(), filters) { + chFiles <- fileInfoType{fileInfo.Name(), dpath} + } + } + } + return nil +} func (sc *SyncCommand) ReadLocalFileKeys(chFiles <-chan fileInfoType, chFinish chan<- error, keys map[string]string) { totalCount := 0 fmt.Printf("\n") @@ -862,7 +999,7 @@ func (sc *SyncCommand) CheckDestBackupDir(sUrl StorageURLer) error { return nil } -func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) error { +func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string, isSrc bool) error { bucketName := sUrl.(CloudURL).bucket bucket, err := sc.command.ossBucket(bucketName) if err != nil { @@ -872,7 +1009,7 @@ func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) err chFiles := make(chan objectInfoType, ChannelBuf) chFinish := make(chan error, 2) go sc.ReadOssKeys(keys, sUrl, chFiles, chFinish) - go sc.GetOssKeyList(bucket, sUrl, chFiles, chFinish) + go sc.GetOssKeyList(bucket, sUrl, chFiles, chFinish, isSrc) select { case err := <-chFinish: if err != nil { @@ -882,15 +1019,80 @@ func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) err return nil } -func (sc *SyncCommand) GetOssKeyList(bucket *oss.Bucket, sURL StorageURLer, chObjects chan<- objectInfoType, chFinish chan<- error) { +func (sc *SyncCommand) GetOssKeyList(bucket *oss.Bucket, sURL StorageURLer, chObjects chan<- objectInfoType, chFinish chan<- error, isSrc bool) { cloudURL := sURL.(CloudURL) - err := getObjectListCommon(bucket, cloudURL, chObjects, sc.syncOption.onlyCurrentDir, - sc.syncOption.filters, sc.syncOption.payerOptions) + err := sc.getObjectListCommon(bucket, cloudURL, chObjects, sc.syncOption.onlyCurrentDir, + sc.syncOption.filters, sc.syncOption.payerOptions, isSrc) if err != nil { chFinish <- err } } +func (sc *SyncCommand) getObjectListCommon(bucket *oss.Bucket, cloudURL CloudURL, chObjects chan<- objectInfoType, + onlyCurrentDir bool, filters []filterOptionType, payerOptions []oss.Option, isSrc bool) error { + defer close(chObjects) + pre := oss.Prefix(cloudURL.object) + marker := oss.Marker("") + //while the src object is end with "/", use object key as marker, exclude the object itself + if strings.HasSuffix(cloudURL.object, "/") { + marker = oss.Marker(cloudURL.object) + } + del := oss.Delimiter("") + if onlyCurrentDir { + del = oss.Delimiter("/") + } + + listOptions := append(payerOptions, pre, marker, del, oss.MaxKeys(1000)) + for { + lor, err := bucket.ListObjects(listOptions...) + if err != nil { + return err + } + + for _, object := range lor.Objects { + if isSrc { + next := sc.command.filterObject(object) + if !next { + continue + } + } + prefix := "" + relativeKey := object.Key + index := strings.LastIndex(cloudURL.object, "/") + if index > 0 { + prefix = object.Key[:index+1] + relativeKey = object.Key[index+1:] + } + + if doesSingleObjectMatchPatterns(object.Key, filters) { + if strings.ToLower(object.Type) == "symlink" { + props, err := bucket.GetObjectDetailedMeta(object.Key, payerOptions...) + if err != nil { + LogError("ossGetObjectStatRetry error info:%s\n", err.Error()) + return err + } + size, err := strconv.ParseInt(props.Get(oss.HTTPHeaderContentLength), 10, 64) + if err != nil { + LogError("strconv.ParseInt error info:%s\n", err.Error()) + return err + + } + object.Size = size + } + chObjects <- objectInfoType{prefix, relativeKey, int64(object.Size), object.LastModified} + } + } + + pre = oss.Prefix(lor.Prefix) + marker = oss.Marker(lor.NextMarker) + listOptions = append(payerOptions, pre, marker, oss.MaxKeys(1000)) + if !lor.IsTruncated { + break + } + } + return nil +} + func (sc *SyncCommand) ReadOssKeys(keys map[string]string, sURL StorageURLer, chObjects <-chan objectInfoType, chFinish chan<- error) { totalCount := 0 fmt.Printf("\n") @@ -941,7 +1143,7 @@ func (sc *SyncCommand) readDirLimit(dirName string, limitCount int) ([]os.FileIn return list, nil } func (sc *SyncCommand) movePath(srcName, destName string) error { - err := sc.moveFileToPath(srcName,destName) + err := sc.moveFileToPath(srcName, destName) if err != nil { LogError("rename %s %s error,%s\n", srcName, destName, err.Error()) } else { @@ -951,11 +1153,11 @@ func (sc *SyncCommand) movePath(srcName, destName string) error { } return err } -func (sc *SyncCommand)moveFileToPath(srcName, destName string) error { - err := os.Rename(srcName,destName) +func (sc *SyncCommand) moveFileToPath(srcName, destName string) error { + err := os.Rename(srcName, destName) if err == nil { return nil - }else{ + } else { inputFile, err := os.Open(srcName) defer inputFile.Close() if err != nil { @@ -976,4 +1178,4 @@ func (sc *SyncCommand)moveFileToPath(srcName, destName string) error { } return nil } -} \ No newline at end of file +} diff --git a/lib/sync_test.go b/lib/sync_test.go index 1d0887a0..320ea668 100644 --- a/lib/sync_test.go +++ b/lib/sync_test.go @@ -2235,3 +2235,159 @@ func (s *OssutilCommandSuite) TestSyncUploadSubSymlinkDir(c *C) { os.RemoveAll(dirName) s.removeBucket(bucketName, true, c) } + +func (s *OssutilCommandSuite) TestSyncWithSizeTimeFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, randLowStr(150), c) + + testFileName3 := dirName + string(os.PathSeparator) + "dest-" + fileName + ".txt" + s.createFile(testFileName3, randLowStr(200), c) + + syncArgs := []string{dirName, CloudURLToString(bucketName, dirName)} + str := "" + cpDir := CheckpointDir + bForce := true + bDelete := true + backupDir := "test-backup-dir" + routines := strconv.Itoa(Routines) + minSize := int64(100) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + "delete": &bDelete, + "backupDir": &backupDir, + OptionMinSize: &minSize, + } + + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + //check, txt object not exist + _, err = s.rawGetStat(bucketName, strings.Replace(testFileName1, string(os.PathSeparator), "/", -1)) + c.Assert(err, NotNil) + + //check, jpg object exist + objectStat := s.getStat(bucketName, strings.Replace(testFileName2, string(os.PathSeparator), "/", -1), c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + objectStat = s.getStat(bucketName, strings.Replace(testFileName3, string(os.PathSeparator), "/", -1), c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + syncArgs = []string{CloudURLToString(bucketName, dirName), dirName} + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + //check, jpg file exist + _, err = os.Stat(testFileName1) + c.Assert(err, NotNil) + + //check, jpg file exist + _, err = os.Stat(testFileName2) + c.Assert(err, IsNil) + + _, err = os.Stat(testFileName3) + c.Assert(err, IsNil) + + backupFile := backupDir + string(os.PathSeparator) + fileName + ".txt" + _, err = os.Stat(backupFile) + c.Assert(err, IsNil) + + os.RemoveAll(backupDir) + s.clearObjects(bucketName, "", c) + + s.createFile(testFileName1, text, c) + maxSize := int64(200) + options[OptionMaxSize] = &maxSize + syncArgs = []string{dirName, CloudURLToString(bucketName, dirName)} + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + //check, txt object not exist + _, err = s.rawGetStat(bucketName, strings.Replace(testFileName1, string(os.PathSeparator), "/", -1)) + c.Assert(err, NotNil) + + objectStat = s.getStat(bucketName, strings.Replace(testFileName2, string(os.PathSeparator), "/", -1), c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + _, err = s.rawGetStat(bucketName, strings.Replace(testFileName3, string(os.PathSeparator), "/", -1)) + c.Assert(err, NotNil) + + maxTime := time.Now().Add(+10 * time.Second).Unix() + c.Log(maxTime) + minTime := time.Now().Add(-10 * time.Second).Unix() + + options[OptionStartTime] = &minTime + options[OptionEndTime] = &maxTime + + syncArgs = []string{CloudURLToString(bucketName, dirName), CloudURLToString(bucketNameDest, dirName)} + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + objectStat = s.getStat(bucketNameDest, strings.Replace(testFileName2, string(os.PathSeparator), "/", -1), c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + s.clearObjects(bucketName, "", c) + s.clearObjects(testFileName2, "", c) + + delete(options, OptionStartTime) + delete(options, OptionEndTime) + delete(options, OptionMinSize) + delete(options, OptionMaxSize) + + syncArgs = []string{dirName, CloudURLToString(bucketName, dirName)} + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + options[OptionStartTime] = &minTime + options[OptionEndTime] = &maxTime + options[OptionMinSize] = &minSize + options[OptionMaxSize] = &maxSize + syncArgs = []string{CloudURLToString(bucketName, dirName), dirName} + _, err = cm.RunCommand("sync", syncArgs, options) + c.Assert(err, IsNil) + + //check, jpg file exist + _, err = os.Stat(testFileName1) + c.Assert(err, NotNil) + + //check, jpg file exist + _, err = os.Stat(testFileName2) + c.Assert(err, IsNil) + + _, err = os.Stat(testFileName3) + c.Assert(err, NotNil) + + backupFile3 := backupDir + string(os.PathSeparator) + "dest-" + fileName + ".txt" + _, err = os.Stat(backupFile3) + c.Assert(err, IsNil) + + backupFile1 := backupDir + string(os.PathSeparator) + fileName + ".txt" + _, err = os.Stat(backupFile1) + c.Assert(err, IsNil) + + os.RemoveAll(backupDir) + s.clearObjects(bucketName, "", c) + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} diff --git a/lib/util.go b/lib/util.go index d31cf536..9a1df2f4 100644 --- a/lib/util.go +++ b/lib/util.go @@ -4,14 +4,13 @@ import ( "bytes" "fmt" "hash" - "io/ioutil" + "io/fs" "math/rand" "os" "os/exec" "os/user" "path/filepath" "runtime" - "strconv" "strings" "time" @@ -533,170 +532,6 @@ func currentHomeDir() string { return homeDir } -func getCurrentDirFileListCommon(dpath string, chFiles chan<- fileInfoType, filters []filterOptionType) error { - if !strings.HasSuffix(dpath, string(os.PathSeparator)) { - dpath += string(os.PathSeparator) - } - - fileList, err := ioutil.ReadDir(dpath) - if err != nil { - return err - } - - for _, fileInfo := range fileList { - if !fileInfo.IsDir() { - realInfo, errF := os.Stat(dpath + fileInfo.Name()) - if errF == nil && realInfo.IsDir() { - // for symlink - continue - } - - if doesSingleFileMatchPatterns(fileInfo.Name(), filters) { - chFiles <- fileInfoType{fileInfo.Name(), dpath} - } - } - } - return nil -} - -func getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir bool, disableAllSymlink bool, - enableSymlinkDir bool, filters []filterOptionType) error { - defer close(chFiles) - if onlyCurrentDir { - return getCurrentDirFileListCommon(dpath, chFiles, filters) - } - - name := dpath - symlinkDiretorys := []string{dpath} - walkFunc := func(fpath string, f os.FileInfo, err error) error { - if f == nil { - return err - } - - dpath = filepath.Clean(dpath) - fpath = filepath.Clean(fpath) - - fileName, err := filepath.Rel(dpath, fpath) - if err != nil { - return fmt.Errorf("list file error: %s, info: %s", fpath, err.Error()) - } - - if f.IsDir() { - if fpath != dpath { - if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { - chFiles <- fileInfoType{fileName, name} - } else { - chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} - } - } - return nil - } - - if disableAllSymlink && (f.Mode()&os.ModeSymlink) != 0 { - return nil - } - - if enableSymlinkDir && (f.Mode()&os.ModeSymlink) != 0 { - // there is difference between os.Stat and os.Lstat in filepath.Walk - realInfo, err := os.Stat(fpath) - if err != nil { - return err - } - - if realInfo.IsDir() { - // it's symlink dir - // if linkDir has suffix os.PathSeparator,os.Lstat determine it is a dir - if !strings.HasSuffix(name, string(os.PathSeparator)) { - name += string(os.PathSeparator) - } - linkDir := name + fileName + string(os.PathSeparator) - symlinkDiretorys = append(symlinkDiretorys, linkDir) - return nil - } - } - - if doesSingleFileMatchPatterns(fileName, filters) { - chFiles <- fileInfoType{fileName, name} - } - return nil - } - - var err error - for { - symlinks := symlinkDiretorys - symlinkDiretorys = []string{} - for _, v := range symlinks { - err = filepath.Walk(v, walkFunc) - if err != nil { - return err - } - } - if len(symlinkDiretorys) == 0 { - break - } - } - return err -} - -func getObjectListCommon(bucket *oss.Bucket, cloudURL CloudURL, chObjects chan<- objectInfoType, - onlyCurrentDir bool, filters []filterOptionType, payerOptions []oss.Option) error { - defer close(chObjects) - pre := oss.Prefix(cloudURL.object) - marker := oss.Marker("") - //while the src object is end with "/", use object key as marker, exclude the object itself - if strings.HasSuffix(cloudURL.object, "/") { - marker = oss.Marker(cloudURL.object) - } - del := oss.Delimiter("") - if onlyCurrentDir { - del = oss.Delimiter("/") - } - - listOptions := append(payerOptions, pre, marker, del, oss.MaxKeys(1000)) - for { - lor, err := bucket.ListObjects(listOptions...) - if err != nil { - return err - } - - for _, object := range lor.Objects { - prefix := "" - relativeKey := object.Key - index := strings.LastIndex(cloudURL.object, "/") - if index > 0 { - prefix = object.Key[:index+1] - relativeKey = object.Key[index+1:] - } - - if doesSingleObjectMatchPatterns(object.Key, filters) { - if strings.ToLower(object.Type) == "symlink" { - props, err := bucket.GetObjectDetailedMeta(object.Key, payerOptions...) - if err != nil { - LogError("ossGetObjectStatRetry error info:%s\n", err.Error()) - return err - } - size, err := strconv.ParseInt(props.Get(oss.HTTPHeaderContentLength), 10, 64) - if err != nil { - LogError("strconv.ParseInt error info:%s\n", err.Error()) - return err - - } - object.Size = size - } - chObjects <- objectInfoType{prefix, relativeKey, int64(object.Size), object.LastModified} - } - } - - pre = oss.Prefix(lor.Prefix) - marker = oss.Marker(lor.NextMarker) - listOptions = append(payerOptions, pre, marker, oss.MaxKeys(1000)) - if !lor.IsTruncated { - break - } - } - return nil -} - func GetPassword(prompt string) ([]byte, error) { fd := int(os.Stdin.Fd()) if terminal.IsTerminal(fd) { @@ -736,3 +571,63 @@ func AddStringsToOption(params []string, options []oss.Option) ([]oss.Option, er } return options, nil } + +// CheckLocalFile check local file +func CheckLocalFile(filterMap map[string]int64, fileInfo fs.FileInfo) bool { + if len(filterMap) == 0 { + return true + } + if filterMap[OptionEndTime] != 0 { + t := time.Unix(filterMap[OptionEndTime], 0) + if !fileInfo.ModTime().Before(t) { + return false + } + } + if filterMap[OptionStartTime] != 0 { + t := time.Unix(filterMap[OptionStartTime], 0) + if !fileInfo.ModTime().After(t) { + return false + } + } + if filterMap[OptionMaxSize] != 0 { + if !(fileInfo.Size() < filterMap[OptionMaxSize]) { + return false + } + } + if filterMap[OptionMinSize] != 0 { + if !(fileInfo.Size() > filterMap[OptionMinSize]) { + return false + } + } + return true +} + +// CheckObject check oss object +func CheckObject(filterMap map[string]int64, fileInfo oss.ObjectProperties) bool { + if len(filterMap) == 0 { + return true + } + if filterMap[OptionEndTime] != 0 { + t := time.Unix(filterMap[OptionEndTime], 0) + if !fileInfo.LastModified.Before(t) { + return false + } + } + if filterMap[OptionStartTime] != 0 { + t := time.Unix(filterMap[OptionStartTime], 0) + if !fileInfo.LastModified.After(t) { + return false + } + } + if filterMap[OptionMaxSize] != 0 { + if !(fileInfo.Size < filterMap[OptionMaxSize]) { + return false + } + } + if filterMap[OptionMinSize] != 0 { + if !(fileInfo.Size > filterMap[OptionMinSize]) { + return false + } + } + return true +}