From 6476d25c888ce9088df82ac259d364aa0244d93a Mon Sep 17 00:00:00 2001 From: yangpeng <183851063@qq.com> Date: Wed, 12 Oct 2022 14:25:18 +0800 Subject: [PATCH] add filter directory --- lib/command_test.go | 534 +++++++++++++++++++++++++++++++--- lib/cp.go | 58 +++- lib/cp_test.go | 158 +++++++++-- lib/ls.go | 14 +- lib/ls_test.go | 581 ++++++++++++++++++++++++++++++++++++- lib/rm.go | 6 +- lib/rm_test.go | 341 ++++++++++++++++++++++ lib/set_acl.go | 4 + lib/set_acl_test.go | 95 ++++++- lib/set_meta.go | 16 +- lib/set_meta_test.go | 99 ++++++- lib/sync.go | 34 +-- lib/sync_test.go | 663 +++++++++++++++++++++++++++++++++++++++++++ lib/util.go | 263 ++++++++++++++--- 14 files changed, 2680 insertions(+), 186 deletions(-) diff --git a/lib/command_test.go b/lib/command_test.go index db905386..49d5b177 100644 --- a/lib/command_test.go +++ b/lib/command_test.go @@ -880,7 +880,55 @@ func (s *OssutilCommandSuite) createTestFiles(dir, subdir string, c *C, contents return filenames } +func (s *OssutilCommandSuite) createTestFilesDir(dir, subdir string, c *C, contents map[string]string) []string { + // Create dirs + err := os.MkdirAll(dir, 0755) + c.Assert(err, IsNil) + + err = os.MkdirAll(dir+string(os.PathSeparator)+subdir, 0755) + c.Assert(err, IsNil) + filenames := make([]string, 0) + + // Create files + num := 3 + for i := 1020; i < 1020+num; i++ { + filename := subdir + "/" + fmt.Sprintf("testfile%d.txt", i) + content := fmt.Sprintf("include测试文件：%d内容", i) + filenames = append(filenames, filename) + contents[filename] = content + filename = dir + "/" + filename + s.createFile(filename, content, c) + } + + num = 2 + for i := 2; i < 2+num; i++ { + filename := subdir + "/" + fmt.Sprintf("test%d.jpg", i) + content := fmt.Sprintf("include test jpg\n%dcontent", i) + filenames = append(filenames, filename) + contents[filename] = content + filename = dir + "/" + filename + s.createFile(filename, content, c) + } + + num = 2 + for i := 103; i < 103+num; i++ { + filename := fmt.Sprintf("testfile%d.txt", i) + content := fmt.Sprintf("include测试文件：%d内容", i) + filenames = append(filenames, filename) + contents[filename] = content + filename = dir + "/" + filename + s.createFile(filename, content, c) + } + filename := subdir + "/" + "my.rtf" + content := fmt.Sprintf("include测试文件：%s内容", "my.rtf") + filenames = append(filenames, filename) + contents[filename] = content + filename = dir + "/" + filename + s.createFile(filename, content, c) + + return filenames +} func (s *OssutilCommandSuite) prepareTestFiles(dir, subdir, filePrefix, fileSuffix string, num int, c *C) []string { // Create dirs f, err := os.Stat(dir) @@ -2001,7 +2049,12 @@ func matchFiltersForFiles(files []fileInfoType, filters []filterOptionType) []fi for i, filter := range filters { if filter.name == IncludePrompt { - res := filterFilesWithInclude(files, filter.pattern) + res := make([]fileInfoType, 0) + if filter.isDir { + res = filterFilesDirWithInclude(files, filter.pattern) + } else { + res = filterFilesWithInclude(files, filter.pattern) + } for _, v := range res { if containsInFileSlice(vsf, v) { continue @@ -2009,11 +2062,20 @@ func matchFiltersForFiles(files []fileInfoType, filters []filterOptionType) []fi vsf = append(vsf, v) } } else { - if i == 0 { - vsf = append(vsf, filterFilesWithExclude(files, filter.pattern)...) + if filter.isDir { + if i == 0 { + vsf = append(vsf, filterFilesDirWithExclude(files, filter.pattern)...) + } else { + vsf = filterFilesDirWithExclude(vsf, filter.pattern) + } } else { - vsf = filterFilesWithExclude(vsf, filter.pattern) + if i == 0 { + vsf = append(vsf, filterFilesWithExclude(files, filter.pattern)...) + } else { + vsf = filterFilesWithExclude(vsf, filter.pattern) + } } + } } @@ -2179,7 +2241,7 @@ func (s *OssutilCommandSuite) TestFilterObjectsFromChanWithPattern(c *C) { func (s *OssutilCommandSuite) TestGetFilter(c *C) { //e.g., ossutil cp oss://tempb4/ . -rf --include "*.txt" --exclude "*.jpg" cmdline := []string{"ossutil", "cp", "oss://tempb4", ".", "-rf", "--include", "*.txt", "--exclude", "*.jpg"} - expect := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*.jpg"}} + expect := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*.jpg", false}} res, fts := getFilter(cmdline) same := reflect.DeepEqual(fts, expect) c.Assert(same, Equals, true) @@ -2268,6 +2330,23 @@ func (s *OssutilCommandSuite) TestFilterSingleStr(c *C) { c.Assert(res, Equals, true) } +func (s *OssutilCommandSuite) TestFilterSingleDir(c *C) { + res := filterSingleDir("dir1/dir2/test18.txt", "dir1/*", true) + c.Assert(res, Equals, true) + + res = filterSingleDir("dir1/dir2/test18.txt", "*dir2/*", true) + c.Assert(res, Equals, true) + + res = filterSingleDir("dir1/dir2/test18.txt", "dir2/*", true) + c.Assert(res, Equals, false) + + res = filterSingleDir("dir1/dir2/test39.txt", "dir1/dir2/*", false) + c.Assert(res, Equals, false) + + res = filterSingleDir("dir1/dir2/test28.txt", "demo/*", false) + c.Assert(res, Equals, true) +} + func (s *OssutilCommandSuite) TestFilterStrsWithInclude(c *C) { strs := []string{"test11.jpg", "test18.txt", "test28.txt", "testfile"} @@ -2307,7 +2386,7 @@ func (s *OssutilCommandSuite) TestFilterStrsWithExclude(c *C) { } func (s *OssutilCommandSuite) TestMatchFiltersForStr(c *C) { - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} res := matchFiltersForStr("test18.txt", fts) c.Assert(res, Equals, true) res = matchFiltersForStr("test28.txt", fts) @@ -2317,7 +2396,7 @@ func (s *OssutilCommandSuite) TestMatchFiltersForStr(c *C) { res = matchFiltersForStr("testfile", fts) c.Assert(res, Equals, false) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} res = matchFiltersForStr("test11.jpg", fts) c.Assert(res, Equals, true) res = matchFiltersForStr("testfile", fts) @@ -2327,7 +2406,7 @@ func (s *OssutilCommandSuite) TestMatchFiltersForStr(c *C) { res = matchFiltersForStr("test28.txt", fts) c.Assert(res, Equals, false) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} res = matchFiltersForStr("test18.txt", fts) c.Assert(res, Equals, true) res = matchFiltersForStr("test11.jpg", fts) @@ -2348,22 +2427,54 @@ func (s *OssutilCommandSuite) TestMatchFiltersForStr(c *C) { c.Assert(res, Equals, true) } +func (s *OssutilCommandSuite) TestMatchFiltersForStrDir(c *C) { + fts := []filterOptionType{{"--include", "dir1/*", true}} + res := matchFiltersForStr("dir1/test18.txt", fts) + c.Assert(res, Equals, true) + res = matchFiltersForStr("dir1/dir2/test28.txt", fts) + c.Assert(res, Equals, true) + res = matchFiltersForStr("dir3/dir2/dir3344/test11.jpg", fts) + c.Assert(res, Equals, false) + res = matchFiltersForStr("dir2/dir4/testfile", fts) + c.Assert(res, Equals, false) + + fts = []filterOptionType{{"--exclude", "*dir2/*", true}} + res = matchFiltersForStr("dir1/dir2/test11.jpg", fts) + c.Assert(res, Equals, false) + res = matchFiltersForStr("die/dir2/testfile", fts) + c.Assert(res, Equals, false) + res = matchFiltersForStr("/hi11/hi22/dir33/test18.txt", fts) + c.Assert(res, Equals, true) + res = matchFiltersForStr("hi/s/s/s/test28.txt", fts) + c.Assert(res, Equals, true) + + fts = []filterOptionType{{"--include", "/dir2/dir3/*", true}, {"--exclude", "/dir2/dir3/dir4/*", true}} + res = matchFiltersForStr("/dir2/dir3/test18.txt", fts) + c.Assert(res, Equals, true) + res = matchFiltersForStr("/dir2/dir3/dir4/test11.jpg", fts) + c.Assert(res, Equals, false) + res = matchFiltersForStr("/dir2/dir3/dir4/test28.txt", fts) + c.Assert(res, Equals, false) + res = matchFiltersForStr("/dir2/dir3/dir4/testfile", fts) + c.Assert(res, Equals, false) +} + func (s *OssutilCommandSuite) TestMatchFiltersForStrs(c *C) { strs := []string{"test11.jpg", "test18.txt", "test28.txt", "testfile"} - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} res := matchFiltersForStrs(strs, fts) expect := []string{"test18.txt", "test28.txt"} same := reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} res = matchFiltersForStrs(strs, fts) expect = []string{"test11.jpg", "testfile"} same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} res = matchFiltersForStrs(strs, fts) expect = []string{"test18.txt"} same = reflect.DeepEqual(res, expect) @@ -2376,22 +2487,56 @@ func (s *OssutilCommandSuite) TestMatchFiltersForStrs(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestMatchFiltersForStrsDir(c *C) { + strs := []string{"/dir1/dir2/dir3/test11.jpg", "/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt", "dir1/dir2/dir3/testfile", "hi1/dir2/hi3/testhihi"} + + fts := []filterOptionType{{"--include", "/dir1/*", true}} + res := matchFiltersForStrs(strs, fts) + expect := []string{"/dir1/dir2/dir3/test11.jpg", "/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt"} + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--exclude", "/dir1/*", true}} + res = matchFiltersForStrs(strs, fts) + expect = []string{"dir1/dir2/dir3/testfile"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "*dir2/*", true}, {"--exclude", "*hi3/*", false}} + res = matchFiltersForStrs(strs, fts) + expect = []string{"/dir1/dir2/dir3/test11.jpg", "/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt", "dir1/dir2/dir3/testfile"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "*dir2/*", true}, {"--exclude", "*hi3/*", false}} + res = matchFiltersForStrs(strs, fts) + expect = []string{"/dir1/dir2/dir3/test11.jpg", "/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt", "dir1/dir2/dir3/testfile"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "/dir1/*.jpg", true}} + res = matchFiltersForStrs(strs, fts) + expect = []string{"/dir1/dir2/dir3/test11.jpg"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestMatchFiltersForStrsInArray(c *C) { strs := []string{"test11.jpg", "test18.txt", "test28.txt", "testfile"} - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} res := matchFiltersForStrsInArray(strs, fts) expect := []string{"test18.txt", "test28.txt"} same := reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} res = matchFiltersForStrsInArray(strs, fts) expect = []string{"test11.jpg", "testfile"} same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} res = matchFiltersForStrsInArray(strs, fts) expect = []string{"test18.txt"} same = reflect.DeepEqual(res, expect) @@ -2404,6 +2549,28 @@ func (s *OssutilCommandSuite) TestMatchFiltersForStrsInArray(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestMatchFiltersForDirsInArray(c *C) { + strs := []string{"/dir1/dir2/dir3/test11.jpg", "/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt", "dir1/dir2/dir3/testfile", "hi1/dir2/hi3/testhihi"} + + fts := []filterOptionType{{"--include", "*dir3/*.txt", true}} + res := matchFiltersForStrsInArray(strs, fts) + expect := []string{"/dir1/dir2/dir3/test11/test18.txt", "/dir1/dir2/dir3/test28.txt"} + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--exclude", "*dir3/*.txt", true}} + res = matchFiltersForStrsInArray(strs, fts) + expect = []string{"/dir1/dir2/dir3/test11.jpg", "dir1/dir2/dir3/testfile", "hi1/dir2/hi3/testhihi"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "*dir2/*", true}, {"--exclude", "*dir3/*.txt", true}} + res = matchFiltersForStrsInArray(strs, fts) + expect = []string{"/dir1/dir2/dir3/test11.jpg", "dir1/dir2/dir3/testfile", "hi1/dir2/hi3/testhihi"} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestDoesSingleFileMatchPatterns(c *C) { filename := "testfile12.jpg" fts := []filterOptionType{} @@ -2411,22 +2578,22 @@ func (s *OssutilCommandSuite) TestDoesSingleFileMatchPatterns(c *C) { c.Assert(res, Equals, true) filename = "dir1/testfile1.txt" - fts = []filterOptionType{{"--include", "*.txt"}} + fts = []filterOptionType{{"--include", "*.txt", false}} res = doesSingleFileMatchPatterns(filename, fts) c.Assert(res, Equals, true) filename = "dir1/testfile2.txt" - fts = []filterOptionType{{"--include", "*.jpg"}} + fts = []filterOptionType{{"--include", "*.jpg", false}} res = doesSingleFileMatchPatterns(filename, fts) c.Assert(res, Equals, false) filename = "dir1/testfile3.txt" - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} res = doesSingleFileMatchPatterns(filename, fts) c.Assert(res, Equals, false) filename = "dir1/testfile3.txt" - fts = []filterOptionType{{"--include", "*.txt"}, {"--include", "*.jpg"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--include", "*.jpg", false}, {"--exclude", "*2*", false}} res = doesSingleFileMatchPatterns(filename, fts) c.Assert(res, Equals, true) } @@ -2451,7 +2618,7 @@ func (s *OssutilCommandSuite) TestGetFilesFromChanToArray(c *C) { c.Assert(same, Equals, true) } -func (s *OssutilCommandSuite) TestcontainsInFileSlice(c *C) { +func (s *OssutilCommandSuite) TestContainsInFileSlice(c *C) { files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} tar := fileInfoType{"dir1/my.rtf", "testdir/"} @@ -2493,6 +2660,35 @@ func (s *OssutilCommandSuite) TestFilterFilesWithInclude(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestFilterFilesDirWithInclude(c *C) { + files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + + expect := []fileInfoType{{"dir1/my.rtf", "testdir/"}} + res := filterFilesDirWithInclude(files, "*dir1/*.rtf") + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{} + res = filterFilesDirWithInclude(files, "dir1/*.jpg") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + res = filterFilesDirWithInclude(files, "*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + res = filterFilesDirWithInclude(files, "testdir/*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{} + res = filterFilesDirWithInclude(files, "") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestFilterFilesWithExclude(c *C) { files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} @@ -2512,6 +2708,30 @@ func (s *OssutilCommandSuite) TestFilterFilesWithExclude(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestFilterFilesDirWithExclude(c *C) { + files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + + expect := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + res := filterFilesDirWithExclude(files, "testdir2/*") + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{} + res = filterFilesDirWithExclude(files, "testdir/*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{} + res = filterFilesDirWithExclude(files, "*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + res = filterFilesDirWithExclude(files, "") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestMatchFiltersForFiles(c *C) { files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} @@ -2521,19 +2741,47 @@ func (s *OssutilCommandSuite) TestMatchFiltersForFiles(c *C) { same := reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}} + fts = []filterOptionType{{"--include", "*.txt", false}} + res = matchFiltersForFiles(files, fts) + expect = []fileInfoType{{"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--exclude", "*.txt", false}} + res = matchFiltersForFiles(files, fts) + expect = []fileInfoType{{"dir1/my.rtf", "testdir/"}} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} + res = matchFiltersForFiles(files, fts) + expect = []fileInfoType{{"dir1/testfile103.txt", "testdir/"}} + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + +func (s *OssutilCommandSuite) TestMatchFiltersForFilesWithDir(c *C) { + files := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + + fts := []filterOptionType{} + res := matchFiltersForFiles(files, fts) + expect := []fileInfoType{{"dir1/my.rtf", "testdir/"}, {"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "testdir/*.txt", true}} res = matchFiltersForFiles(files, fts) expect = []fileInfoType{{"dir1/testfile103.txt", "testdir/"}, {"testfile1021.txt", "testdir/"}} same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "testdir/*.txt", true}} res = matchFiltersForFiles(files, fts) expect = []fileInfoType{{"dir1/my.rtf", "testdir/"}} same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "testdir/*.txt", true}, {"--exclude", "testdir/*2*", true}} res = matchFiltersForFiles(files, fts) expect = []fileInfoType{{"dir1/testfile103.txt", "testdir/"}} same = reflect.DeepEqual(res, expect) @@ -2560,7 +2808,7 @@ func (s *OssutilCommandSuite) TestFilterFileFromChanWithPatterns(c *C) { chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} close(chFiles) - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} dstFiles := make(chan fileInfoType, ChannelBuf) filterFilesFromChanWithPatterns(chFiles, fts, dstFiles) c.Assert(len(dstFiles), Equals, 2) @@ -2570,7 +2818,7 @@ func (s *OssutilCommandSuite) TestFilterFileFromChanWithPatterns(c *C) { chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} close(chFiles) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} dstFiles2 := make(chan fileInfoType, ChannelBuf) filterFilesFromChanWithPatterns(chFiles, fts, dstFiles2) c.Assert(len(dstFiles2), Equals, 1) @@ -2580,7 +2828,39 @@ func (s *OssutilCommandSuite) TestFilterFileFromChanWithPatterns(c *C) { chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} close(chFiles) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} + dstFiles3 := make(chan fileInfoType, ChannelBuf) + filterFilesFromChanWithPatterns(chFiles, fts, dstFiles3) + c.Assert(len(dstFiles3), Equals, 1) +} + +func (s *OssutilCommandSuite) TestFilterFileFromChanWithPatternsDir(c *C) { + chFiles := make(chan fileInfoType, ChannelBuf) + chFiles <- fileInfoType{"dir1/my.rtf", "testdir/"} + chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} + chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} + close(chFiles) + fts := []filterOptionType{{"--include", "testdir/*.txt", true}} + dstFiles := make(chan fileInfoType, ChannelBuf) + filterFilesFromChanWithPatterns(chFiles, fts, dstFiles) + c.Assert(len(dstFiles), Equals, 2) + + chFiles = make(chan fileInfoType, ChannelBuf) + chFiles <- fileInfoType{"dir1/my.rtf", "testdir/"} + chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} + chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} + close(chFiles) + fts = []filterOptionType{{"--exclude", "testdir/*.txt", true}} + dstFiles2 := make(chan fileInfoType, ChannelBuf) + filterFilesFromChanWithPatterns(chFiles, fts, dstFiles2) + c.Assert(len(dstFiles2), Equals, 1) + + chFiles = make(chan fileInfoType, ChannelBuf) + chFiles <- fileInfoType{"dir1/my.rtf", "testdir/"} + chFiles <- fileInfoType{"dir1/testfile103.txt", "testdir/"} + chFiles <- fileInfoType{"testfile1021.txt", "testdir/"} + close(chFiles) + fts = []filterOptionType{{"--include", "testdir/*.txt", true}, {"--exclude", "testdir/*2*", true}} dstFiles3 := make(chan fileInfoType, ChannelBuf) filterFilesFromChanWithPatterns(chFiles, fts, dstFiles3) c.Assert(len(dstFiles3), Equals, 1) @@ -2593,22 +2873,49 @@ func (s *OssutilCommandSuite) TestDoesSingleObjectMatchPatterns(c *C) { c.Assert(res, Equals, true) object = "dir1/testfile1.txt" - fts = []filterOptionType{{"--include", "*.txt"}} + fts = []filterOptionType{{"--include", "*.txt", false}} res = doesSingleObjectMatchPatterns(object, fts) c.Assert(res, Equals, true) object = "dir1/testfile2.txt" - fts = []filterOptionType{{"--include", "*.jpg"}} + fts = []filterOptionType{{"--include", "*.jpg", false}} res = doesSingleObjectMatchPatterns(object, fts) c.Assert(res, Equals, false) object = "dir1/testfile3.txt" - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} res = doesSingleObjectMatchPatterns(object, fts) c.Assert(res, Equals, false) object = "dir1/testfile4.txt" - fts = []filterOptionType{{"--include", "*.txt"}, {"--include", "*.jpg"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--include", "*.jpg", false}, {"--exclude", "*2*", false}} + res = doesSingleObjectMatchPatterns(object, fts) + c.Assert(res, Equals, true) +} + +func (s *OssutilCommandSuite) TestDoesSingleObjectMatchPatternsDir(c *C) { + object := "dir1/testfile3.jpg" + fts := []filterOptionType{} + res := doesSingleObjectMatchPatterns(object, fts) + c.Assert(res, Equals, true) + + object = "dir1/testfile1.txt" + fts = []filterOptionType{{"--include", "dir1/*.txt", true}} + res = doesSingleObjectMatchPatterns(object, fts) + c.Assert(res, Equals, true) + + object = "dir1/testfile2.txt" + fts = []filterOptionType{{"--include", "dir1/*.jpg", true}} + res = doesSingleObjectMatchPatterns(object, fts) + c.Assert(res, Equals, false) + + object = "dir1/testfile3.txt" + fts = []filterOptionType{{"--exclude", "dir1/*.txt", true}} + res = doesSingleObjectMatchPatterns(object, fts) + c.Assert(res, Equals, false) + + object = "dir1/testfile4.txt" + fts = []filterOptionType{{"--include", "dir1/*.txt", true}, {"--include", "dir1/*.jpg", false}, {"--exclude", "dir1/*2*", true}} res = doesSingleObjectMatchPatterns(object, fts) c.Assert(res, Equals, true) } @@ -2669,6 +2976,43 @@ func (s *OssutilCommandSuite) TestFilterObjectsWithInclude(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestFilterObjectsWithIncludeDir(c *C) { + objects := []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + + expect := []objectInfoType{{"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}} + res := filterObjectsWithIncludeDir(objects, "dir1/*.rtf") + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{} + res = filterObjectsWithIncludeDir(objects, "dir1/*.jpg") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + res = filterObjectsWithIncludeDir(objects, "*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{} + res = filterObjectsWithIncludeDir(objects, "") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}} + res = filterObjectsWithIncludeDir(objects, "dir1/test*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestFilterObjectsWithExclude(c *C) { objects := []objectInfoType{ {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, @@ -2699,6 +3043,43 @@ func (s *OssutilCommandSuite) TestFilterObjectsWithExclude(c *C) { c.Assert(same, Equals, true) } +func (s *OssutilCommandSuite) TestFilterObjectsWithExcludeDir(c *C) { + objects := []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + + expect := []objectInfoType{ + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + res := filterObjectsWithExcludeDir(objects, "dir1/*.rtf") + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{} + res = filterObjectsWithExcludeDir(objects, "*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + res = filterObjectsWithExcludeDir(objects, "") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + expect = []objectInfoType{ + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + res = filterObjectsWithExcludeDir(objects, "dir1/*") + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + func (s *OssutilCommandSuite) TestMatchFiltersForObjects(c *C) { objects := []objectInfoType{ {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, @@ -2716,27 +3097,75 @@ func (s *OssutilCommandSuite) TestMatchFiltersForObjects(c *C) { same := reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}} + fts = []filterOptionType{{"--include", "*.txt", false}} + res = matchFiltersForObjects(objects, fts) + expect = []objectInfoType{ + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--exclude", "*.txt", false}} + res = matchFiltersForObjects(objects, fts) + expect = []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + } + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} res = matchFiltersForObjects(objects, fts) expect = []objectInfoType{ {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + } + same = reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) +} + +func (s *OssutilCommandSuite) TestMatchFiltersForObjectsWithDir(c *C) { + objects := []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + {"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + + fts := []filterOptionType{{"--include", "*", true}} + res := matchFiltersForObjects(objects, fts) + expect := []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + {"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + } + same := reflect.DeepEqual(res, expect) + c.Assert(same, Equals, true) + + fts = []filterOptionType{{"--include", "dir1/*.txt", true}} + res = matchFiltersForObjects(objects, fts) + expect = []objectInfoType{ + {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, } same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "dir1/*.txt", true}} res = matchFiltersForObjects(objects, fts) expect = []objectInfoType{ {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, + {"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, + {"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, } same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "dir1/*", true}, {"--exclude", "*2*", true}} res = matchFiltersForObjects(objects, fts) expect = []objectInfoType{ + {"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)}, {"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)}, } same = reflect.DeepEqual(res, expect) c.Assert(same, Equals, true) @@ -2766,7 +3195,7 @@ func (s *OssutilCommandSuite) TestFilterObjectFromChanWithPatterns(c *C) { chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} close(chObjects) - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} dstObjects := make(chan objectInfoType, ChannelBuf) filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects) c.Assert(len(dstObjects), Equals, 2) @@ -2776,7 +3205,7 @@ func (s *OssutilCommandSuite) TestFilterObjectFromChanWithPatterns(c *C) { chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} close(chObjects) - fts = []filterOptionType{{"--exclude", "*.txt"}} + fts = []filterOptionType{{"--exclude", "*.txt", false}} dstObjects2 := make(chan objectInfoType, ChannelBuf) filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects2) c.Assert(len(dstObjects2), Equals, 1) @@ -2786,8 +3215,43 @@ func (s *OssutilCommandSuite) TestFilterObjectFromChanWithPatterns(c *C) { chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} close(chObjects) - fts = []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts = []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} dstObjects3 := make(chan objectInfoType, ChannelBuf) filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects3) c.Assert(len(dstObjects3), Equals, 1) } + +func (s *OssutilCommandSuite) TestFilterObjectFromChanWithPatternsDir(c *C) { + chObjects := make(chan objectInfoType, ChannelBuf) + chObjects <- objectInfoType{"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)} + chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} + chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + chObjects <- objectInfoType{"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + close(chObjects) + fts := []filterOptionType{{"--include", "dir1/*.txt", true}} + dstObjects := make(chan objectInfoType, ChannelBuf) + filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects) + c.Assert(len(dstObjects), Equals, 1) + + chObjects = make(chan objectInfoType, ChannelBuf) + chObjects <- objectInfoType{"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)} + chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} + chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + chObjects <- objectInfoType{"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + close(chObjects) + fts = []filterOptionType{{"--exclude", "dir1/*.txt", true}} + dstObjects2 := make(chan objectInfoType, ChannelBuf) + filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects2) + c.Assert(len(dstObjects2), Equals, 3) + + chObjects = make(chan objectInfoType, ChannelBuf) + chObjects <- objectInfoType{"dir1/", "my.rtf", 10240, time.Date(2017, 10, 1, 7, 0, 0, 0, time.Local)} + chObjects <- objectInfoType{"dir1/", "testfile103.txt", 1040, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)} + chObjects <- objectInfoType{"", "testfile1021.txt", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + chObjects <- objectInfoType{"dir1/", "demo.jpg", 1024, time.Date(2017, 1, 19, 7, 10, 35, 0, time.UTC)} + close(chObjects) + fts = []filterOptionType{{"--include", "dir1/*", true}, {"--exclude", "dir1/*3*", true}} + dstObjects3 := make(chan objectInfoType, ChannelBuf) + filterObjectsFromChanWithPatterns(chObjects, fts, dstObjects3) + c.Assert(len(dstObjects3), Equals, 2) +} diff --git a/lib/cp.go b/lib/cp.go index 55fd8d24..e093d6a5 100644 --- a/lib/cp.go +++ b/lib/cp.go @@ -69,6 +69,7 @@ type copyOptionType struct { type filterOptionType struct { name string pattern string + isDir bool } type fileInfoType struct { @@ -210,7 +211,6 @@ var specChineseCopy = SpecText{ ?：匹配单个字符 [sequence]：匹配sequence的任意字符 [!sequence]：匹配不在sequence的任意字符 - 注意：规则不支持带目录的格式，e.g.，--include "/usr/*/test/*.jpg"。 --include和--exclude可以出现多次。当多个规则出现时，这些规则按从左往右的顺序应用。例如： 当前目录下包含3个文件： @@ -229,6 +229,23 @@ var specChineseCopy = SpecText{ $ ossutil cp . oss://my-bucket/path --exclude '*.jpg' --include 'testfile*.jpg' --exclude 'testfile?.jpg' 上传testfile2.txt到oss://my-bucket/path/testfile2.txt 上传testfile33.jpg到oss://my-bucket/path/testfile33.jpg + + 当前目录下包含3个文件一个目录： + testfile1.jpg + testfiel2.txt + testfile33.jpg + testdir1 + -testfile1.txt + -testfile2.txt + -testfile3.txt + $ ossutil cp . oss://my-bucket/path --include 'testdir1/*' + 上传testdir1目录下的文件到oss://my-bucket/path/testdir1/ + + $ ossutil cp . oss://my-bucket/path --exclude 'testdir1/*' + 上传testfile1.jpg到oss://my-bucket/path/testfile1.jpg + 上传testfile2.txt到oss://my-bucket/path/testfile2.txt + 上传testfile33.jpg到oss://my-bucket/path/testfile33.jpg + --meta选项 @@ -736,7 +753,6 @@ var specEnglishCopy = SpecText{ ?: Matches any single character [sequence]: Matches any character in sequence [!sequence]: Matches any character not in sequence - Note: does not support patterns containing directory info. e.g., --include "/usr/*/test/*.jpg" Any number of these parameters can be passed to a command. You can do this by providing an --exclude or --include argument multiple times, e.g., @@ -767,6 +783,22 @@ var specEnglishCopy = SpecText{ upload testfile2.txt to oss://my-bucket/path/testfile2.txt upload testfile33.jpg to oss://my-bucket/path/testfile33.jpg + e.g., 3 files in current dir and 1 sub directory + testfile1.jpg + testfiel2.txt + testfile33.jpg + testdir1 + -testfile1.txt + -testfile2.txt + -testfile3.txt + $ ossutil cp . oss://my-bucket/path --include 'testdir1/*' + upload files in testdir1 directory to oss://my-bucket/path/testdir1/ + + $ ossutil cp . oss://my-bucket/path --exclude 'testdir1/*' + upload testfile1.jpg to oss://my-bucket/path/testfile1.jpg + upload testfile2.txt to oss://my-bucket/path/testfile2.txt + upload testfile33.jpg to oss://my-bucket/path/testfile33.jpg + --meta option This option will set the specified objects' meta data. If --recursive option is specified, @@ -1769,7 +1801,10 @@ func (cc *CopyCommand) getFileListStatistic(dpath string) error { if f.IsDir() { if fpath != dpath { - cc.monitor.updateScanNum(1) + if doesSingleFileMatchPatterns(fpath, cc.cpOption.filters) { + cc.monitor.updateScanNum(1) + } + } return nil } @@ -1803,7 +1838,7 @@ func (cc *CopyCommand) getFileListStatistic(dpath string) error { return nil } } - if doesSingleFileMatchPatterns(f.Name(), cc.cpOption.filters) { + if doesSingleFileMatchPatterns(fpath, cc.cpOption.filters) { cc.monitor.updateScanSizeNum(realFileSize, 1) } return nil @@ -1903,11 +1938,14 @@ func (cc *CopyCommand) getFileList(dpath string, chFiles chan<- fileInfoType) er if f.IsDir() { if fpath != dpath { - if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { - chFiles <- fileInfoType{fileName, name} - } else { - chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} + if doesSingleFileMatchPatterns(fpath, cc.cpOption.filters) { + if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { + chFiles <- fileInfoType{fileName, name} + } else { + chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} + } } + } return nil } @@ -1934,8 +1972,7 @@ func (cc *CopyCommand) getFileList(dpath string, chFiles chan<- fileInfoType) er return nil } } - - if doesSingleFileMatchPatterns(fileName, cc.cpOption.filters) { + if doesSingleFileMatchPatterns(fpath, cc.cpOption.filters) { chFiles <- fileInfoType{fileName, name} } return nil @@ -2369,7 +2406,6 @@ func (cc *CopyCommand) downloadFiles(srcURL CloudURL, destURL FileURL) error { prefix = srcURL.object[:index+1] relativeKey = srcURL.object[index+1:] } - go cc.objectStatistic(bucket, srcURL) err := cc.downloadSingleFileWithReport(bucket, objectInfoType{prefix, relativeKey, -1, time.Now()}, filePath) return cc.formatResultPrompt(err) diff --git a/lib/cp_test.go b/lib/cp_test.go index 4edd6488..f9b82bc3 100644 --- a/lib/cp_test.go +++ b/lib/cp_test.go @@ -1787,6 +1787,56 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithNormalInclude(c *C) { s.removeBucket(bucketName, true, c) } +// Test: --inlcude 'dir/dir1/*' +func (s *OssutilCommandSuite) TestBatchCPObjectWithNormalIncludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "testdir-inc1" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + // upload files + // e.g., ossutil cp testdir-inc/ oss://tempb4 -rf --include 'dir/dir1/*' + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include", dir + "/" + subdir + "/*"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // download above files with --include uploaded + // e.g., ossutil cp oss://tempb4/ testdownload/ -rf --include "dir/dir1/*" + downdir := "testdownload-inc1" + randLowStr(5) + args = []string{bucketStr, downdir} + cmdline = []string{"ossutil", "cp", bucketStr, downdir, "-rf", "--include", dir + "/" + subdir + "/*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // Get uploaded files (with above conditions: --include dir+"/"+subdir+"/*") and use these for verification + files := filterDirsWithInclude(filenames, dir+"/"+subdir+"/*") + + // Verify + _, err = os.Stat(downdir) + c.Assert(err, IsNil) + + for _, filename := range files { + tname := downdir + "/" + filename + _, err := os.Stat(tname) + c.Assert(err, IsNil) + + content := s.readFile(tname, c) + c.Assert(content, Equals, contents[filename]) + } + + // cleanup + os.RemoveAll(dir) + os.RemoveAll(downdir) + s.removeBucket(bucketName, true, c) +} + // Test: --include '*2??txt' func (s *OssutilCommandSuite) TestBatchCPObjectWithMarkInclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) @@ -2135,6 +2185,56 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithNormalExclude(c *C) { s.removeBucket(bucketName, true, c) } +// Test: --exclude 'dir/dir1/*' +func (s *OssutilCommandSuite) TestBatchCPObjectWithNormalExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "testdir-exc1" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + // upload files + // e.g., ossutil cp testdir-exc/ oss://tempb4 -rf --exclude dir+"/"+subdir+"/*" + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf", "--exclude", dir + "/" + subdir + "/*"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // download above files with --exclude uploaded + // e.g., ossutil cp oss://tempb4/ testdownload/ -rf --exclude dir+"/"+subdir+"/*" + downdir := "testdownload-exc1" + randLowStr(5) + args = []string{bucketStr, downdir} + cmdline = []string{"ossutil", "cp", bucketStr, downdir, "-rf", "--exclude", dir + "/" + subdir + "/*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // Get uploaded files (with above conditions: --exclude dir+"/"+subdir+"/*") and use these for verification + files := filterDirsWithExclude(filenames, subdir+"/*") + + // Verify + _, err = os.Stat(downdir) + c.Assert(err, IsNil) + + for _, filename := range files { + tname := downdir + "/" + filename + _, err := os.Stat(tname) + c.Assert(err, IsNil) + + content := s.readFile(tname, c) + c.Assert(content, Equals, contents[filename]) + } + + // cleanup + os.RemoveAll(dir) + os.RemoveAll(downdir) + s.removeBucket(bucketName, true, c) +} + // Test: --exclude '*2??txt' func (s *OssutilCommandSuite) TestBatchCPObjectWithMarkExclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) @@ -2466,7 +2566,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiNormalIncludeExclude(c * c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} files := matchFiltersForStrs(filenames, fts) // Verify @@ -2517,7 +2617,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiRepeatedIncludeExclude(c c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}, {"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}, {"--include", "*.txt", false}, {"--exclude", "*2*", false}} files := matchFiltersForStrs(filenames, fts) // Verify @@ -2568,7 +2668,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiFullIncludeExclude(c *C) c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--include", "*"}, {"--exclude", "*"}} + fts := []filterOptionType{{"--include", "*", false}, {"--exclude", "*", false}} files := matchFiltersForStrs(filenames, fts) c.Assert(len(files), Equals, 0) @@ -2621,7 +2721,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiFullExcludeInclude(c *C) c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--exclude", "*"}, {"--include", "*"}} + fts := []filterOptionType{{"--exclude", "*", false}, {"--include", "*", false}} files := matchFiltersForStrs(filenames, fts) c.Assert(len(files), Equals, len(filenames)) @@ -2674,10 +2774,9 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithInvalidIncludeExclude(c *C) { c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "cp", dir, bucketStr, "-f", "--include", "/*.txt", "--exclude", "*2*"} - showElapse, err = s.rawCPWithFilter(args, false, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include", "/*.txt", "--exclude", "*2*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(showElapse, Equals, true) // download // e.g., ossutil cp oss://tempb4/ testdownload/ -f --exclude "*.txt" @@ -2693,10 +2792,9 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithInvalidIncludeExclude(c *C) { c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "cp", dir, bucketStr, "-f", "--include", "*.txt", "--exclude", "/usr/*/*2*"} - showElapse, err = s.rawCPWithFilter(args, false, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include", "*.txt", "--exclude", "/usr/*/*2*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(showElapse, Equals, true) // download test with --meta, --acl cmdline = []string{"ossutil", "cp", bucketStr, downdir, "-rf", "--meta", "Cache-Control:no-cache"} @@ -2751,7 +2849,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiFullExcludeIncludeEqual( c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--exclude", "*"}, {"--include", "*"}} + fts := []filterOptionType{{"--exclude", "*", false}, {"--include", "*", false}} files := matchFiltersForStrs(filenames, fts) c.Assert(len(files), Equals, len(filenames)) @@ -2803,10 +2901,9 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithInvalidIncludeExcludeEqual(c c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "cp", dir, bucketStr, "-f", "--include=/*.txt", "--exclude=*2*"} - showElapse, err = s.rawCPWithFilter(args, false, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include=/*.txt", "--exclude=*2*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(showElapse, Equals, true) // download // e.g., ossutil cp oss://tempb4/ testdownload/ -f --exclude "*.txt" @@ -2822,10 +2919,9 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithInvalidIncludeExcludeEqual(c c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "cp", dir, bucketStr, "-f", "--include=*.txt", "--exclude=/usr/*/*2*"} - showElapse, err = s.rawCPWithFilter(args, false, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include=*.txt", "--exclude=/usr/*/*2*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(showElapse, Equals, true) // download test with --meta, --acl cmdline = []string{"ossutil", "cp", bucketStr, downdir, "-rf", "--meta", "Cache-Control:no-cache"} @@ -2882,10 +2978,10 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiNormalOnlyIncludeEqual(c c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include="*.txt") and use these for verification - fts := []filterOptionType{{"--include", "*.txt"}} + fts := []filterOptionType{{"--include", "*.txt", false}} files := matchFiltersForStrs(filenames, fts) - ftsJpg := []filterOptionType{{"--include", "*.jpg"}} + ftsJpg := []filterOptionType{{"--include", "*.jpg", false}} filesJpg := matchFiltersForStrs(filenames, ftsJpg) c.Assert(len(filesJpg) > 0, Equals, true) @@ -2943,14 +3039,14 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiNormalOnlyExcludeEqual(c c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - ftsTxt := []filterOptionType{{"--include", "*.txt"}} + ftsTxt := []filterOptionType{{"--include", "*.txt", false}} filesTxt := matchFiltersForStrs(filenames, ftsTxt) - ftsJpg := []filterOptionType{{"--include", "*.jpg"}} + ftsJpg := []filterOptionType{{"--include", "*.jpg", false}} filesJpg := matchFiltersForStrs(filenames, ftsJpg) c.Assert(len(filesJpg) > 0, Equals, true) - ftsRtf := []filterOptionType{{"--include", "*.rtf"}} + ftsRtf := []filterOptionType{{"--include", "*.rtf", false}} filesRtf := matchFiltersForStrs(filenames, ftsRtf) c.Assert(len(filesRtf) > 0, Equals, true) @@ -3014,14 +3110,14 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMultiNormalIncludeMixtureExcl c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - ftsTxt := []filterOptionType{{"--include", "*.txt"}} + ftsTxt := []filterOptionType{{"--include", "*.txt", false}} filesTxt := matchFiltersForStrs(filenames, ftsTxt) - ftsJpg := []filterOptionType{{"--include", "*.jpg"}} + ftsJpg := []filterOptionType{{"--include", "*.jpg", false}} filesJpg := matchFiltersForStrs(filenames, ftsJpg) c.Assert(len(filesJpg) > 0, Equals, true) - ftsRtf := []filterOptionType{{"--include", "*.rtf"}} + ftsRtf := []filterOptionType{{"--include", "*.rtf", false}} filesRtf := matchFiltersForStrs(filenames, ftsRtf) c.Assert(len(filesRtf) > 0, Equals, true) @@ -3088,7 +3184,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectWithMetaAcl(c *C) { c.Assert(showElapse, Equals, true) // Get uploaded files (with above conditions: --include "*.txt" --include "*.jpg" --exclude "*2*") and use these for verification - fts := []filterOptionType{{"--include", "*.txt"}, {"--include", "*.jpg"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--include", "*.jpg", false}, {"--exclude", "*2*", false}} files := matchFiltersForStrs(filenames, fts) // Verify @@ -3154,7 +3250,7 @@ func (s *OssutilCommandSuite) TestBatchCPObjectBetweenOssWithMetaAcl(c *C) { c.Assert(showElapse, Equals, true) // Get files (with above conditions: --include "*" --exclude "*10*") and use these for verification - fts := []filterOptionType{{"--include", "*"}, {"--exclude", "*10*"}} + fts := []filterOptionType{{"--include", "*", false}, {"--exclude", "*10*", false}} files := matchFiltersForStrs(filenames, fts) // Verify diff --git a/lib/ls.go b/lib/ls.go index 8667aaa9..6a2c7989 100644 --- a/lib/ls.go +++ b/lib/ls.go @@ -39,7 +39,6 @@ var specChineseList = SpecText{ ?：匹配单个字符 [sequence]：匹配sequence的任意字符 [!sequence]：匹配不在sequence的任意字符 - 注意：规则不支持带目录的格式，e.g.，--include "/usr/*/test/*.jpg"。 --include和--exclude可以出现多次。当多个规则出现时，这些规则按从左往右的顺序应用 @@ -169,10 +168,10 @@ var specChineseList = SpecText{ Object Number is: 1 0.066567(s) elapsed - 14) ossutil ls oss://bucket --include "*.avi" --include "*.mp4" --exclude "*.png" --exclude "*.jpg" + 14) ossutil ls oss://bucket --include "*.avi" --include "dir1/*" --exclude "*.png" --exclude "*.jpg" LastModifiedTime Size(B) StorageClass ETAG ObjectName - 2019-05-30 14:23:51 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/test.avi - 2019-05-30 14:24:05 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/test.mp4 + 2019-05-30 14:23:51 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/dir1/test.avi + 2019-05-30 14:24:05 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/dir1/test.mp4 Object Number is: 2 15) ossutil ls oss://bucket --all-versions @@ -214,7 +213,6 @@ var specEnglishList = SpecText{ ?: Matches any single character [sequence]: Matches any character in sequence [!sequence]: Matches any character not in sequence - Note: does not support patterns containing directory info. e.g., --include "/usr/*/test/*.jpg" Any number of these parameters can be passed to a command. You can do this by providing an --exclude or --include argument multiple times, e.g., @@ -351,10 +349,10 @@ Usage: Object Number is: 1 0.066567(s) elapsed - 14) ossutil ls oss://bucket --include "*.avi" --include "*.mp4" --exclude "*.png" --exclude "*.jpg" + 14) ossutil ls oss://bucket --include "*.avi" --include "dir1/*" --exclude "*.png" --exclude "*.jpg" LastModifiedTime Size(B) StorageClass ETAG ObjectName - 2019-05-30 14:23:51 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/test.avi - 2019-05-30 14:24:05 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/test.mp4 + 2019-05-30 14:23:51 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/dir1/test.avi + 2019-05-30 14:24:05 +0800 CST 1030 Standard 4A902D176BE0EE4224BC196BBB8CCC69 oss://bucket/dir1/test.mp4 Object Number is: 2 15) ossutil ls oss://bucket[/prefix] --all-versions `, diff --git a/lib/ls_test.go b/lib/ls_test.go index 4cd0ce22..3df1f7e0 100644 --- a/lib/ls_test.go +++ b/lib/ls_test.go @@ -719,7 +719,6 @@ func (s *OssutilCommandSuite) TestListObjectfilterInclude(c *C) { subdir := "dir1" contents := map[string]string{} filenames := s.createTestFiles(dir, subdir, c, contents) - // upload files args := []string{dir, bucketStr} cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} @@ -777,6 +776,78 @@ func (s *OssutilCommandSuite) TestListObjectfilterInclude(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestFilterDirsWithInclude(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "ossutil-test-dir-" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFilesDir(dir, subdir, c, contents) + // 2 txt dir1 3 txt 2 jpg 1 rft + testLogger.Printf("filenames:%#v\n", filenames) + // upload files + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // ls files + // e.g., ossutil ls oss://tempb4/ --include "*dir1/*.jpg" --include "*dir1/*.txt" + cmdline = []string{"ossutil", "ls", bucketStr, "--include", "*dir1/*.jpg", "--include", "*dir1/*.txt"} + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + files := filterDirsWithInclude(filenames, "*dir1/*.jpg") + testLogger.Printf("files:%#v\n", files) + c.Assert(len(files), Equals, 2) + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + files = filterDirsWithInclude(filenames, "*dir1/*.txt") + c.Assert(len(files), Equals, 3) + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + files = filterDirsWithInclude(filenames, "*dir1/*.rtf") + c.Assert(len(files), Equals, 1) + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestListObjectfilterExclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -843,6 +914,73 @@ func (s *OssutilCommandSuite) TestListObjectfilterExclude(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestListObjectfilterExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "ossutil-test-dir-" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFilesDir(dir, subdir, c, contents) + + // upload files + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + // ls files + cmdline = []string{"ossutil", "ls", bucketStr, "--exclude", "dir1/*.jpg"} + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + } + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + // 2 txt dir1 3 txt 2 jpg 1 rft + // Verify + files := filterDirsWithInclude(filenames, "*dir1/*.jpg") + c.Assert(len(files), Equals, 2) + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + files = filterDirsWithExclude(filenames, "*dir1/*.jpg") + c.Assert(len(files), Equals, 6) + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + files = filterDirsWithExclude(filenames, "*dir1/*.rtf") + c.Assert(len(files), Equals, 7) + + files = filterDirsWithExclude(filenames, "*dir1/*") + c.Assert(len(files), Equals, 2) + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestListDirectoryfilterInclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -908,12 +1046,177 @@ func (s *OssutilCommandSuite) TestListDirectoryfilterInclude(c *C) { c.Assert(err, IsNil) // Verify - c.Assert(strings.Contains(string(fileBody), dir1), Equals, true) + c.Assert(strings.Contains(string(fileBody), dir1), Equals, true) + for _, filename := range filenames1 { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + c.Assert(strings.Contains(string(fileBody), dir2), Equals, false) + for _, filename := range filenames2 { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir1) + os.RemoveAll(dir2) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestListDirectoryFilterIncludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + // directory1 + dir1 := "ossutil-test-dir-" + randLowStr(5) + subdir1 := "dir1" + contents1 := map[string]string{} + filenames1 := s.createTestFiles(dir1, subdir1, c, contents1) + + // directory2 + dir2 := "ossutil-test-dir-" + randLowStr(5) + subdir2 := "dir2" + contents2 := map[string]string{} + filenames2 := s.createTestFiles(dir2, subdir2, c, contents2) + + // upload directory1 + // e.g., ossutil cp dir1 oss://tempb4/dir1 -rf + args := []string{dir1, bucketStr + "/" + dir1} + cmdline := []string{"ossutil", "cp", dir1, bucketStr + "/" + dir1, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // upload directory2 + // e.g., ossutil cp dir2 oss://tempb4/dir2 -rf + args = []string{dir2, bucketStr + "/" + dir2} + cmdline = []string{"ossutil", "cp", dir2, bucketStr + "/" + dir2, "-rf"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // ls files + // e.g., ossutil ls oss://tempb4/ --include dir1/* + cmdline = []string{bucketStr, "--include"} + strFilter := dir1 + "/*" + cmdline = append(cmdline, strFilter) + lsArgs := []string{CloudURLToString(bucketName, "")} + limitedNum := strconv.FormatInt(-1, 10) + str := "" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), dir1), Equals, true) + for _, filename := range filenames1 { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + c.Assert(strings.Contains(string(fileBody), dir2), Equals, false) + for _, filename := range filenames2 { + if strings.Contains(filename, subdir2) { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } else { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + } + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir1) + os.RemoveAll(dir2) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestListDirectoryfilterExclude(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + // directory1 + dir1 := "ossutil-test-dir-" + randLowStr(5) + subdir1 := "dir1" + contents1 := map[string]string{} + filenames1 := s.createTestFiles(dir1, subdir1, c, contents1) + + // directory2 + dir2 := "ossutil-test-dir-" + randLowStr(5) + subdir2 := "dir2" + contents2 := map[string]string{} + filenames2 := s.createTestFiles(dir2, subdir2, c, contents2) + + // upload directory1 + args := []string{dir1, bucketStr + "/" + dir1} + cmdline := []string{"ossutil", "cp", dir1, bucketStr + "/" + dir1, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // upload directory2 + args = []string{dir2, bucketStr + "/" + dir2} + cmdline = []string{"ossutil", "cp", dir2, bucketStr + "/" + dir2, "-rf"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // ls files + // e.g., ossutil ls oss://tempb4/ --include "*.jpg" --exclude dir1 + cmdline = []string{"ossutil", "ls", bucketStr, "--exclude"} + strFilter := "*" + dir1 + cmdline = append(cmdline, strFilter) + + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bDirectory := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "directory": &bDirectory, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), dir1), Equals, false) for _, filename := range filenames1 { c.Assert(strings.Contains(string(fileBody), filename), Equals, false) } - c.Assert(strings.Contains(string(fileBody), dir2), Equals, false) + c.Assert(strings.Contains(string(fileBody), dir2), Equals, true) for _, filename := range filenames2 { c.Assert(strings.Contains(string(fileBody), filename), Equals, false) } @@ -925,7 +1228,7 @@ func (s *OssutilCommandSuite) TestListDirectoryfilterInclude(c *C) { s.removeBucket(bucketName, true, c) } -func (s *OssutilCommandSuite) TestListDirectoryfilterExclude(c *C) { +func (s *OssutilCommandSuite) TestListDirectoryFilterExcludeDir(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) bucketStr := CloudURLToString(bucketName, "") @@ -957,9 +1260,9 @@ func (s *OssutilCommandSuite) TestListDirectoryfilterExclude(c *C) { c.Assert(showElapse, Equals, true) // ls files - // e.g., ossutil ls oss://tempb4/ --include "*.jpg" --include dir1 + // e.g., ossutil ls oss://tempb4/ --exclude dir1 cmdline = []string{"ossutil", "ls", bucketStr, "--exclude"} - strFilter := "*" + dir1 + strFilter := dir1 cmdline = append(cmdline, strFilter) limitedNum := strconv.FormatInt(-1, 10) @@ -989,6 +1292,7 @@ func (s *OssutilCommandSuite) TestListDirectoryfilterExclude(c *C) { fileBody, err := ioutil.ReadFile(testOutFileName) c.Assert(err, IsNil) + testLogger.Printf("fileBody:%#v\n", string(fileBody)) // Verify c.Assert(strings.Contains(string(fileBody), dir1), Equals, false) for _, filename := range filenames1 { @@ -1063,6 +1367,62 @@ func (s *OssutilCommandSuite) TestListPartfilterInclude(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestListPartFilterIncludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + bucket, err := makeBucketCommand.command.ossBucket(bucketName) + + // object jpg + object1 := "ossutil-test-object/dir1/" + randLowStr(5) + ".jpg" + _, err = bucket.InitiateMultipartUpload(object1) + c.Assert(err, IsNil) + + // object png + object2 := "ossutil-test-object/dir2/" + randLowStr(5) + ".png" + _, err = bucket.InitiateMultipartUpload(object2) + c.Assert(err, IsNil) + + // ls files + cmdline := []string{"ossutil", "ls", "-m", bucketStr, "--include", "ossutil-test-object/dir1/*"} + + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bPart := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "multipart": &bPart, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), object1), Equals, true) + c.Assert(strings.Contains(string(fileBody), object2), Equals, false) + + // cleanup + os.Remove(testOutFileName) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestListPartfilterExclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -1119,6 +1479,62 @@ func (s *OssutilCommandSuite) TestListPartfilterExclude(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestListPartFilterExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + bucket, err := makeBucketCommand.command.ossBucket(bucketName) + + // object jpg + object1 := "ossutil-test-object/dir1/" + randLowStr(5) + ".jpg" + _, err = bucket.InitiateMultipartUpload(object1) + c.Assert(err, IsNil) + + // object png + object2 := "ossutil-test-object/dir2/" + randLowStr(5) + ".png" + _, err = bucket.InitiateMultipartUpload(object2) + c.Assert(err, IsNil) + + // ls files + cmdline := []string{"ossutil", "ls", "-m", bucketStr, "--exclude", "ossutil-test-object/dir1/*"} + + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bPart := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "multipart": &bPart, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), object1), Equals, false) + c.Assert(strings.Contains(string(fileBody), object2), Equals, true) + + // cleanup + os.Remove(testOutFileName) + s.removeBucket(bucketName, true, c) +} + // list objects versions func (s *OssutilCommandSuite) TestListObjectVersionsNormal(c *C) { bucketName := bucketNamePrefix + randLowStr(10) @@ -1226,7 +1642,6 @@ func (s *OssutilCommandSuite) TestListObjectVersionsNormal(c *C) { s.removeBucket(bucketName, true, c) } -// list objects versions // list objects versions func (s *OssutilCommandSuite) TestListObjectVersionsMarker(c *C) { bucketName := bucketNamePrefix + randLowStr(10) @@ -1395,6 +1810,70 @@ func (s *OssutilCommandSuite) TestListObjectfilterIncludeVersions(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestListObjectfilterIncludeVersionsDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + s.putBucketVersioning(bucketName, string(oss.VersionEnabled), c) + + bucketStr := CloudURLToString(bucketName, "") + + dir := "ossutil-test-dir-" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + // upload files + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // "rm oss://bucket/prefix -r -f" + // remove object + s.removeObjects(bucketName, "re", true, true, c) + + // ls files + // e.g., ossutil ls oss://tempb4/ --include dir/subdir + cmdline = []string{"ossutil", "ls", bucketStr, "--include", dir + "/" + subdir} + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + allVersions := true + str := "" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "allVersions": &allVersions, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + files := filterDirsWithInclude(filenames, dir+"/"+subdir+"/*") + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} func (s *OssutilCommandSuite) TestListDirectoryfilterIncludeVersions(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -1484,3 +1963,91 @@ func (s *OssutilCommandSuite) TestListDirectoryfilterIncludeVersions(c *C) { os.RemoveAll(dir2) s.removeBucket(bucketName, true, c) } + +func (s *OssutilCommandSuite) TestListDirectoryFilterIncludeVersionsDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + s.putBucketVersioning(bucketName, string(oss.VersionEnabled), c) + + bucketStr := CloudURLToString(bucketName, "") + + // directory1 + dir1 := "ossutil-test-dir-" + randLowStr(5) + subdir1 := "dir1" + contents1 := map[string]string{} + filenames1 := s.createTestFiles(dir1, subdir1, c, contents1) + + // directory2 + dir2 := "ossutil-test-dir-" + randLowStr(5) + subdir2 := "dir2" + contents2 := map[string]string{} + filenames2 := s.createTestFiles(dir2, subdir2, c, contents2) + + testLogger.Printf("filenames2:%#v\n", filenames2) + // upload directory1 + args := []string{dir1, bucketStr + "/" + dir1} + cmdline := []string{"ossutil", "cp", dir1, bucketStr + "/" + dir1, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // upload directory2 + args = []string{dir2, bucketStr + "/" + dir2} + cmdline = []string{"ossutil", "cp", dir2, bucketStr + "/" + dir2, "-rf"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // "rm oss://bucket/prefix -r -f" + // remove object + s.removeObjects(bucketName, "re", true, true, c) + + // ls files + // e.g., ossutil ls oss://tempb4/ --include dir1 + cmdline = []string{"ossutil", "ls", bucketStr, "--include"} + strFilter := dir1 + cmdline = append(cmdline, strFilter) + + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bDirectory := true + allVersions := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "directory": &bDirectory, + "allVersions": &allVersions, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), dir1), Equals, true) + for _, filename := range filenames1 { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + c.Assert(strings.Contains(string(fileBody), dir2), Equals, false) + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir1) + os.RemoveAll(dir2) + s.removeBucket(bucketName, true, c) +} diff --git a/lib/rm.go b/lib/rm.go index 44e0987b..7ef12e9c 100644 --- a/lib/rm.go +++ b/lib/rm.go @@ -70,7 +70,6 @@ var specChineseRemove = SpecText{ ?：匹配单个字符 [sequence]：匹配sequence的任意字符 [!sequence]：匹配不在sequence的任意字符 - 注意：规则不支持带目录的格式，e.g.，--include "/usr/*/test/*.jpg"。 --include和--exclude可以出现多次。当多个规则出现时，这些规则按从左往右的顺序应用 @@ -136,7 +135,7 @@ var specChineseRemove = SpecText{ ossutil rm oss://bucket2 -r -b -f ossutil rm oss://bucket2 -a -r -b -f ossutil rm oss://bucket2/%e4%b8%ad%e6%96%87 --encoding-type url - ossutil rm oss://bucket1/objdir -r --include "*.jpg" --include "*.png" --exclude "*.avi" --exclude "*.mp4" + ossutil rm oss://bucket1/objdir -r --include "*.jpg" --include "dir1/*" --exclude "*.avi" --exclude "*.mp4" ossutil rm oss://bucket1/obj1 --version-id versionId ossutil rm oss://bucket1/obj1 --all-versions ossutil rm oss://bucket1/objdir -r --all-versions @@ -198,7 +197,6 @@ var specEnglishRemove = SpecText{ ?: Matches any single character [sequence]: Matches any character in sequence [!sequence]: Matches any character not in sequence - Note: does not support patterns containing directory info. e.g., --include "/usr/*/test/*.jpg" Any number of these parameters can be passed to a command. You can do this by providing an --exclude or --include argument multiple times, e.g., @@ -279,7 +277,7 @@ Usage: ossutil rm oss://bucket2 -r -b -f ossutil rm oss://bucket2 -a -r -b -f ossutil rm oss://bucket2/%e4%b8%ad%e6%96%87 --encoding-type url - ossutil rm oss://bucket1/objdir -r --include "*.jpg" --include "*.png" --exclude "*.avi" --exclude "*.mp4" + ossutil rm oss://bucket1/objdir -r --include "*.jpg" --include "dir1/*" --exclude "*.avi" --exclude "*.mp4" ossutil rm oss://bucket1/obj1 --version-id versionId ossutil rm oss://bucket1/obj1 --all-versions ossutil rm oss://bucket1/objdir -r --all-versions diff --git a/lib/rm_test.go b/lib/rm_test.go index 939f55b8..0d094ebe 100644 --- a/lib/rm_test.go +++ b/lib/rm_test.go @@ -503,6 +503,100 @@ func (s *OssutilCommandSuite) TestRmObjectfilter(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestRmObjectfilterDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "ossutil-test-dir-" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + // upload files + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // ls files + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + for _, filename := range filenames { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + // then rm objects + cmdline = []string{"ossutil", "rm", bucketStr, "-rf", "--include", dir + "/" + subdir + "/*"} + rmArgs := []string{CloudURLToString(bucketName, "")} + bRecusive := true + bForce := true + rmOptions := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "recursive": &bRecusive, + "force": &bForce, + } + os.Args = cmdline + _, err = cm.RunCommand("rm", rmArgs, rmOptions) + os.Args = []string{} + c.Assert(err, IsNil) + + // check again after rm + testOutFile, _ = os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout = os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + + fileBody, err = ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify include + files := filterDirsWithInclude(filenames, dir+"/"+subdir+"/*") + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + // Verify exclude + files = filterDirsWithExclude(filenames, dir+"/"+subdir+"/*") + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestRmPartfilter(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -558,6 +652,61 @@ func (s *OssutilCommandSuite) TestRmPartfilter(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestRmPartfilterDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + bucket, err := makeBucketCommand.command.ossBucket(bucketName) + + // object jpg + object1 := "ossutil-test-object/dir1/" + randLowStr(5) + ".jpg" + _, err = bucket.InitiateMultipartUpload(object1) + c.Assert(err, IsNil) + + // object png + object2 := "ossutil-test-object/dir2/" + randLowStr(5) + ".png" + _, err = bucket.InitiateMultipartUpload(object2) + c.Assert(err, IsNil) + + // ls files + cmdline := []string{"ossutil", "ls", "-m", bucketStr, "--include", "ossutil-test-object/dir1/*"} + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bPart := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "multipart": &bPart, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + os.Args = cmdline + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), object1), Equals, true) + c.Assert(strings.Contains(string(fileBody), object2), Equals, false) + + // cleanup + os.Remove(testOutFileName) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestRmPartfilterExclude(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -648,6 +797,96 @@ func (s *OssutilCommandSuite) TestRmPartfilterExclude(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestRmPartfilterExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + bucket, err := makeBucketCommand.command.ossBucket(bucketName) + + // object jpg + object1 := "ossutil-test-object/dir1/" + randLowStr(5) + ".jpg" + _, err = bucket.InitiateMultipartUpload(object1) + c.Assert(err, IsNil) + + // object png + object2 := "ossutil-test-object/dir2/" + randLowStr(5) + ".png" + _, err = bucket.InitiateMultipartUpload(object2) + c.Assert(err, IsNil) + + // ls files + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + bPart := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "multipart": &bPart, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + os.Args = []string{} + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), object1), Equals, true) + c.Assert(strings.Contains(string(fileBody), object2), Equals, true) + + // then rm objects + cmdline := []string{"ossutil", "rm", bucketStr, "-rf", "--include", "ossutil-test-object/dir1/*"} + rmArgs := []string{CloudURLToString(bucketName, "")} + bRecusive := true + bForce := true + bMultiPart := true + rmOptions := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "recursive": &bRecusive, + "force": &bForce, + "multipart": &bMultiPart, + } + + os.Args = cmdline + _, err = cm.RunCommand("rm", rmArgs, rmOptions) + os.Args = []string{} + c.Assert(err, IsNil) + + // check again after rm + testOutFile, _ = os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout = os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + + fileBody, err = ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + c.Assert(strings.Contains(string(fileBody), object1), Equals, false) + c.Assert(strings.Contains(string(fileBody), object2), Equals, true) + + // cleanup + os.Remove(testOutFileName) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestRmSpecialCharacterKey(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -1047,6 +1286,108 @@ func (s *OssutilCommandSuite) TestRmObjectfilterVersioning(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestRmObjectfilterVersioningDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + s.putBucketVersioning(bucketName, "enabled", c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "ossutil-test-dir-" + randLowStr(5) + subdir := "dir1" + contents := map[string]string{} + filenames := s.createTestFiles(dir, subdir, c, contents) + + // upload files + args := []string{dir, bucketStr} + cmdline := []string{"ossutil", "cp", dir, bucketStr, "-rf"} + showElapse, err := s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + // "rm oss://bucket/prefix -r -f" + // remove object,create delete marker + s.removeObjects(bucketName, "re", true, true, c) + + // ls files + limitedNum := strconv.FormatInt(-1, 10) + lsArgs := []string{CloudURLToString(bucketName, "")} + str := "" + allVersions := true + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "limitedNum": &limitedNum, + "allVersions": &allVersions, + } + + testOutFileName := "ossutil-test-outfile-" + randLowStr(5) + testOutFile, _ := os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout := os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + + fileBody, err := ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify + for _, filename := range filenames { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + // then rm objects + cmdline = []string{"ossutil", "rm", bucketStr, "-rf", "--include", dir + "/" + subdir + "/*"} + rmArgs := []string{CloudURLToString(bucketName, "")} + bRecusive := true + bForce := true + rmOptions := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "recursive": &bRecusive, + "force": &bForce, + "allVersions": &allVersions, + } + os.Args = cmdline + _, err = cm.RunCommand("rm", rmArgs, rmOptions) + os.Args = []string{} + c.Assert(err, IsNil) + + // check again after rm + testOutFile, _ = os.OpenFile(testOutFileName, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664) + oldStdout = os.Stdout + os.Stdout = testOutFile + _, err = cm.RunCommand("ls", lsArgs, options) + c.Assert(err, IsNil) + testOutFile.Close() + os.Stdout = oldStdout + + fileBody, err = ioutil.ReadFile(testOutFileName) + c.Assert(err, IsNil) + + // Verify include + files := filterDirsWithInclude(filenames, dir+"/"+subdir+"/*") + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, false) + } + + // Verify exclude + files = filterDirsWithInclude(filenames, dir+"/"+subdir+"/*") + for _, filename := range files { + c.Assert(strings.Contains(string(fileBody), filename), Equals, true) + } + + // cleanup + os.Remove(testOutFileName) + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestRmObjectWithPayer(c *C) { s.createFile(uploadFileName, content, c) bucketName := payerBucket diff --git a/lib/set_acl.go b/lib/set_acl.go index 688debee..eb745982 100644 --- a/lib/set_acl.go +++ b/lib/set_acl.go @@ -121,6 +121,8 @@ ACL： (3)ossutil set-acl oss://bucket1/obj default -r ossutil set-acl oss://bucket1/obj default -r --include "*.jpg" ossutil set-acl oss://bucket1/obj default -r --exclude "*.jpg" + ossutil set-acl oss://bucket1 default -r --include "dir1/*" + ossutil set-acl oss://bucket1 default -r --exclude "dir1/*" (4)ossutil set-acl oss://bucket1/%e4%b8%ad%e6%96%87 default --encoding-type url @@ -204,6 +206,8 @@ Usage： (3)ossutil set-acl oss://bucket1/obj default -r ossutil set-acl oss://bucket1/obj default -r --include "*.jpg" ossutil set-acl oss://bucket1/obj default -r --exclude "*.jpg" + ossutil set-acl oss://bucket1 default -r --include "dir1/*" + ossutil set-acl oss://bucket1 default -r --exclude "dir1/*" (4)ossutil set-acl oss://bucket1/%e4%b8%ad%e6%96%87 default --encoding-type url diff --git a/lib/set_acl_test.go b/lib/set_acl_test.go index fe035ea9..efe8effb 100644 --- a/lib/set_acl_test.go +++ b/lib/set_acl_test.go @@ -52,7 +52,7 @@ func (s *OssutilCommandSuite) TestSetBucketErrorACL(c *C) { func (s *OssutilCommandSuite) TestSetNotExistBucketACL(c *C) { bucketName := bucketNamePrefix + randLowStr(10) - // set bucket acl will be invalid when bucket not exist + // set bucket acl will be invalid when bucket not exist showElapse, err := s.rawSetBucketACL(bucketName, "public-read", true) c.Assert(err, NotNil) @@ -1035,7 +1035,44 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithMultiNormalIncludeExclude(c *C c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} + matchedObjs := matchFiltersForStrs(objs, fts) + + for _, obj := range matchedObjs { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["ACL"], Equals, "public-read") + } + + for _, obj := range objs { + if !containsInStrsSlice(matchedObjs, obj) { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["ACL"], Equals, "default") + } + } + + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + +// Test: --include subdir/* --exclude "*2*" +func (s *OssutilCommandSuite) TestSetObjectAclWithMultiNormalIncludeExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "test-setacl-inc-exc-normal" + subdir := "subdir" + objs := s.createTestObjects(dir, subdir, bucketStr, c) + + acl := "public-read" + args := []string{bucketStr, acl} + cmdline := []string{"ossutil", "set-acl", bucketStr, acl, "-rf", "--include", dir + "/" + subdir + "/*", "--exclude", "*2*"} + + showElapse, err := s.rawSetAclWithFilter(args, true, true, cmdline) + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + fts := []filterOptionType{{"--include", dir + "/" + subdir + "/*", true}, {"--exclude", "*2*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1072,7 +1109,7 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithMultiRepeatedIncludeExclude(c c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}, {"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}, {"--include", "*.txt", false}, {"--exclude", "*2*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1109,7 +1146,7 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithMultiFullIncludeExclude(c *C) c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*"}, {"--exclude", "*"}} + fts := []filterOptionType{{"--include", "*", false}, {"--exclude", "*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1146,7 +1183,44 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithMultiFullExcludeInclude(c *C) c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--exclude", "*"}, {"--include", "*"}} + fts := []filterOptionType{{"--exclude", "*", false}, {"--include", "*", false}} + matchedObjs := matchFiltersForStrs(objs, fts) + + for _, obj := range matchedObjs { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["ACL"], Equals, "public-read") + } + + for _, obj := range objs { + if !containsInStrsSlice(matchedObjs, obj) { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["ACL"], Equals, "default") + } + } + + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + +// Test: --exclude 'dir/subdir/*' --include "dir/subdir/*" +func (s *OssutilCommandSuite) TestSetObjectAclWithMultiFullExcludeIncludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "test-setacl-exc-inc-full" + subdir := "subdir" + objs := s.createTestObjects(dir, subdir, bucketStr, c) + + acl := "public-read" + args := []string{bucketStr, acl} + cmdline := []string{"ossutil", "set-acl", bucketStr, acl, "-rf", "--exclude", dir + "/" + subdir + "/*", "--include", dir + "/" + subdir + "/*"} + + showElapse, err := s.rawSetAclWithFilter(args, true, true, cmdline) + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + fts := []filterOptionType{{"--exclude", dir + "/" + subdir + "/*", true}, {"--include", dir + "/" + subdir + "/*", true}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1194,10 +1268,9 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithInvalidIncExc(c *C) { c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "set-acl", bucketStr, acl, "-f", "--include", "/*.txt", "--exclude", "*2*"} - showElapse, err = s.rawSetAclWithFilter(args, false, true, cmdline) - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "set-acl", bucketStr, acl, "-rf", "--include", "/*.txt", "--exclude", "*2*"} + showElapse, err = s.rawSetAclWithFilter(args, true, true, cmdline) + c.Assert(showElapse, Equals, true) os.RemoveAll(dir) s.removeBucket(bucketName, true, c) @@ -1206,7 +1279,7 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithInvalidIncExc(c *C) { func (s *OssutilCommandSuite) TestSetObjectAclWithVersion(c *C) { bucketName := bucketNamePrefix + "-set-alc-" + randLowStr(10) objectName := randStr(12) - + s.putBucket(bucketName, c) s.putBucketVersioning(bucketName, "enabled", c) @@ -1302,4 +1375,4 @@ func (s *OssutilCommandSuite) TestSetObjectAclWithInvalidVersionArgs(c *C) { args = []string{CloudURLToString(bucketName, objectName), "private"} _, err = cm.RunCommand("set-acl", args, options) c.Assert(strings.Contains(err.Error(), "--version-id only work on single object"), Equals, true) -} \ No newline at end of file +} diff --git a/lib/set_meta.go b/lib/set_meta.go index f0e675ea..40a86a8e 100644 --- a/lib/set_meta.go +++ b/lib/set_meta.go @@ -160,11 +160,11 @@ Headers: (2)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text --update -r 批量更新以o开头的objects的X-Oss-Meta-empty和Content-Type头域 - (3)ossutil set-meta oss://bucket1/ X-Oss-Meta-empty:#Content-Type:plain/text --update -r --include "*.jpg" - 批量更新后缀为.jpg的objects的X-Oss-Meta-empty和Content-Type头域 + (3)ossutil set-meta oss://bucket1/ X-Oss-Meta-empty:#Content-Type:plain/text --update -r --include "dir1/*" --include "*.jpg" + 批量更新dir1目录下且后缀为.jpg的objects的X-Oss-Meta-empty和Content-Type头域 - (4)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text --update -r --exclude "*.jpg" - 批量更新以o开头后缀为.jpg的objects的X-Oss-Meta-empty和Content-Type头域 + (4)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text --update -r --exclude "*.jpg" --exclude "oss/*" + 批量更新以o开头排除oss目录且后缀不为.jpg的objects的X-Oss-Meta-empty和Content-Type头域 (5)ossutil set-meta oss://bucket1/obj1 X-Oss-Meta-delete --delete 删除obj1的X-Oss-Meta-delete头域 @@ -281,11 +281,11 @@ Usage: (2)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text -u -r Batch update X-Oss-Meta-empty and Content-Type header on objects that start with o - (3)ossutil set-meta oss://bucket1/ X-Oss-Meta-empty:#Content-Type:plain/text --update -r --include "*.jpg" - Batch update X-Oss-Meta-empty and Content-Type header on objects ending with .jpg + (3)ossutil set-meta oss://bucket1/ X-Oss-Meta-empty:#Content-Type:plain/text --update -r --include "dir1/*" --include "*.jpg" + Batch update X-Oss-Meta-empty and Content-Type header on objects in dir1 directory ending with .jpg - (4)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text --update -r --exclude ".jpg" - Batch update X-Oss-Meta-empty and Content-Type header on objects starting with o and ending with .jpg + (4)ossutil set-meta oss://bucket1/o X-Oss-Meta-empty:#Content-Type:plain/text --update -r --exclude ".jpg" --exclude "oss/*" + Batch update X-Oss-Meta-empty and Content-Type header on objects starting with o,exclude oss directory and ending not with .jpg (5)ossutil set-meta oss://bucket1/obj1 X-Oss-Meta-delete -d Delete X-Oss-Meta-delete header of obj1 diff --git a/lib/set_meta_test.go b/lib/set_meta_test.go index 8c719b3a..6722dba4 100644 --- a/lib/set_meta_test.go +++ b/lib/set_meta_test.go @@ -1502,7 +1502,91 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiNormalIncludeExclude(c * c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}} + matchedObjs := matchFiltersForStrs(objs, fts) + + for _, obj := range matchedObjs { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["Content-Type"], Equals, "xyz") + c.Assert(objectStat["X-Oss-Meta-Test"], Equals, "with-filter-inc-exc") + } + + for _, obj := range objs { + if !containsInStrsSlice(matchedObjs, obj) { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["Content-Type"] != "xyz", Equals, true) + _, ok := objectStat["X-Oss-Meta-Test"] + c.Assert(ok, Equals, false) + } + } + + // cleanup + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + +// Test: --include 'dir/subdir/*' --exclude "*2*" +func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiNormalIncludeDirExclude(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "test-setmeta-inc-exc-normal" + subdir := "subdir" + objs := s.createTestObjects(dir, subdir, bucketStr, c) + + // Set meta + meta := "content-type:xyz#X-Oss-Meta-test:with-filter-inc-exc" + args := []string{bucketStr, meta} + cmdline := []string{"ossutil", "set-meta", bucketStr, meta, "-rf", "--include", dir + "/" + subdir + "/*", "--exclude", "*2*"} + + showElapse, err := s.rawSetMetaWithFilter(args, true, false, true, true, DefaultLanguage, cmdline) + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + fts := []filterOptionType{{"--include", dir + "/" + subdir + "/*", true}, {"--exclude", "*2*", false}} + matchedObjs := matchFiltersForStrs(objs, fts) + + for _, obj := range matchedObjs { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["Content-Type"], Equals, "xyz") + c.Assert(objectStat["X-Oss-Meta-Test"], Equals, "with-filter-inc-exc") + } + + for _, obj := range objs { + if !containsInStrsSlice(matchedObjs, obj) { + objectStat := s.getStat(bucketName, obj, c) + c.Assert(objectStat["Content-Type"] != "xyz", Equals, true) + _, ok := objectStat["X-Oss-Meta-Test"] + c.Assert(ok, Equals, false) + } + } + + // cleanup + os.RemoveAll(dir) + s.removeBucket(bucketName, true, c) +} + +// Test: --include '*2*' --exclude "dir/subdir/*" +func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiNormalIncludeDirExcludeDir(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + bucketStr := CloudURLToString(bucketName, "") + + dir := "test-setmeta-inc-exc-normal" + subdir := "subdir" + objs := s.createTestObjects(dir, subdir, bucketStr, c) + + // Set meta + meta := "content-type:xyz#X-Oss-Meta-test:with-filter-inc-exc" + args := []string{bucketStr, meta} + cmdline := []string{"ossutil", "set-meta", bucketStr, meta, "-rf", "--include", "*2*", "--exclude", dir + "/" + subdir + "/*"} + + showElapse, err := s.rawSetMetaWithFilter(args, true, false, true, true, DefaultLanguage, cmdline) + c.Assert(err, IsNil) + c.Assert(showElapse, Equals, true) + + fts := []filterOptionType{{"--include", "*2*", false}, {"--exclude", dir + "/" + subdir + "/*", true}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1544,7 +1628,7 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiRepeatedIncludeExclude(c c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*.txt"}, {"--exclude", "*2*"}, {"--include", "*.txt"}, {"--exclude", "*2*"}} + fts := []filterOptionType{{"--include", "*.txt", false}, {"--exclude", "*2*", false}, {"--include", "*.txt", false}, {"--exclude", "*2*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1586,7 +1670,7 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiFullIncludeExclude(c *C) c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--include", "*"}, {"--exclude", "*"}} + fts := []filterOptionType{{"--include", "*", false}, {"--exclude", "*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1628,7 +1712,7 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithMultiFullExcludeInclude(c *C) c.Assert(err, IsNil) c.Assert(showElapse, Equals, true) - fts := []filterOptionType{{"--exclude", "*"}, {"--include", "*"}} + fts := []filterOptionType{{"--exclude", "*", false}, {"--include", "*", false}} matchedObjs := matchFiltersForStrs(objs, fts) for _, obj := range matchedObjs { @@ -1678,10 +1762,9 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithInvalidIncExc(c *C) { c.Assert(showElapse, Equals, false) c.Assert(err.Error() == "--include or --exclude only work with --recursive", Equals, true) - cmdline = []string{"ossutil", "cp", dir, bucketStr, "-f", "--include", "*.txt", "--exclude", "/*2*"} - showElapse, err = s.rawCPWithFilter(args, false, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") - c.Assert(showElapse, Equals, false) - c.Assert(err.Error() == "--include or --exclude does not support format containing dir info", Equals, true) + cmdline = []string{"ossutil", "cp", dir, bucketStr, "-rf", "--include", "*.txt", "--exclude", "/*2*"} + showElapse, err = s.rawCPWithFilter(args, true, true, false, DefaultBigFileThreshold, CheckpointDir, cmdline, "", "") + c.Assert(showElapse, Equals, true) // cleanup os.RemoveAll(dir) diff --git a/lib/sync.go b/lib/sync.go index 959784d8..4d0073e9 100644 --- a/lib/sync.go +++ b/lib/sync.go @@ -547,9 +547,9 @@ func (sc *SyncCommand) RunCommand() error { srcKeys := make(map[string]string) destKeys := make(map[string]string) if srcURL.IsFileURL() { - err = sc.GetLocalFileKeys(srcURL, srcKeys) + err = sc.GetLocalFileKeys(srcURL, srcKeys, false) } else { - err = sc.GetOssKeys(srcURL, srcKeys) + err = sc.GetOssKeys(srcURL, srcKeys, false) } if err != nil { @@ -557,9 +557,9 @@ func (sc *SyncCommand) RunCommand() error { } if destURL.IsFileURL() { - err = sc.GetLocalFileKeys(destURL, destKeys) + err = sc.GetLocalFileKeys(destURL, destKeys, true) } else { - err = sc.GetOssKeys(destURL, destKeys) + err = sc.GetOssKeys(destURL, destKeys, true) } if err != nil { @@ -737,7 +737,7 @@ func (sc *SyncCommand) getCommandType(srcURL StorageURLer, destURL StorageURLer) return operationTypePut } -func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]string) error { +func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]string, dest bool) error { strPath := sUrl.ToString() if !strings.HasSuffix(strPath, string(os.PathSeparator)) { // for symlink dir @@ -747,7 +747,7 @@ func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]strin chFiles := make(chan fileInfoType, ChannelBuf) chFinish := make(chan error, 2) go sc.ReadLocalFileKeys(chFiles, chFinish, keys) - go sc.GetFileList(strPath, chFiles, chFinish) + go sc.GetFileList(strPath, chFiles, chFinish, dest) select { case err := <-chFinish: if err != nil { @@ -757,9 +757,9 @@ func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]strin return nil } -func (sc *SyncCommand) GetFileList(strPath string, chFiles chan<- fileInfoType, chFinish chan<- error) { +func (sc *SyncCommand) GetFileList(strPath string, chFiles chan<- fileInfoType, chFinish chan<- error, dest bool) { err := getFileListCommon(strPath, chFiles, sc.syncOption.onlyCurrentDir, - sc.syncOption.disableAllSymlink, sc.syncOption.enableSymlinkDir, sc.syncOption.filters) + sc.syncOption.disableAllSymlink, sc.syncOption.enableSymlinkDir, sc.syncOption.filters, dest) if err != nil { chFinish <- err } @@ -862,7 +862,7 @@ func (sc *SyncCommand) CheckDestBackupDir(sUrl StorageURLer) error { return nil } -func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) error { +func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string, dest bool) error { bucketName := sUrl.(CloudURL).bucket bucket, err := sc.command.ossBucket(bucketName) if err != nil { @@ -872,7 +872,7 @@ func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) err chFiles := make(chan objectInfoType, ChannelBuf) chFinish := make(chan error, 2) go sc.ReadOssKeys(keys, sUrl, chFiles, chFinish) - go sc.GetOssKeyList(bucket, sUrl, chFiles, chFinish) + go sc.GetOssKeyList(bucket, sUrl, chFiles, chFinish, dest) select { case err := <-chFinish: if err != nil { @@ -882,10 +882,10 @@ func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) err return nil } -func (sc *SyncCommand) GetOssKeyList(bucket *oss.Bucket, sURL StorageURLer, chObjects chan<- objectInfoType, chFinish chan<- error) { +func (sc *SyncCommand) GetOssKeyList(bucket *oss.Bucket, sURL StorageURLer, chObjects chan<- objectInfoType, chFinish chan<- error, dest bool) { cloudURL := sURL.(CloudURL) err := getObjectListCommon(bucket, cloudURL, chObjects, sc.syncOption.onlyCurrentDir, - sc.syncOption.filters, sc.syncOption.payerOptions) + sc.syncOption.filters, sc.syncOption.payerOptions, dest) if err != nil { chFinish <- err } @@ -941,7 +941,7 @@ func (sc *SyncCommand) readDirLimit(dirName string, limitCount int) ([]os.FileIn return list, nil } func (sc *SyncCommand) movePath(srcName, destName string) error { - err := sc.moveFileToPath(srcName,destName) + err := sc.moveFileToPath(srcName, destName) if err != nil { LogError("rename %s %s error,%s\n", srcName, destName, err.Error()) } else { @@ -951,11 +951,11 @@ func (sc *SyncCommand) movePath(srcName, destName string) error { } return err } -func (sc *SyncCommand)moveFileToPath(srcName, destName string) error { - err := os.Rename(srcName,destName) +func (sc *SyncCommand) moveFileToPath(srcName, destName string) error { + err := os.Rename(srcName, destName) if err == nil { return nil - }else{ + } else { inputFile, err := os.Open(srcName) defer inputFile.Close() if err != nil { @@ -976,4 +976,4 @@ func (sc *SyncCommand)moveFileToPath(srcName, destName string) error { } return nil } -} \ No newline at end of file +} diff --git a/lib/sync_test.go b/lib/sync_test.go index 1d0887a0..c391d79b 100644 --- a/lib/sync_test.go +++ b/lib/sync_test.go @@ -848,6 +848,59 @@ func (s *OssutilCommandSuite) TestSyncUploadIncludeFilterSuccess(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestSyncUploadIncludeDirFilterSuccess(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // sync dir to oss + syncArgs := []string{dirName, CloudURLToString(bucketName, "")} + cmdline := []string{"ossutil", "sync", dirName, CloudURLToString(bucketName, ""), "-f", "--include", dirName + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, get stat + objectStat := s.getStat(bucketName, object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + objectStat = s.getStat(bucketName, object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestSyncDownloadIncludeFilter(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -930,6 +983,88 @@ func (s *OssutilCommandSuite) TestSyncDownloadIncludeFilter(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestSyncDownloadIncludeDirFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := dirName + "/" + fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := dirName + "/" + fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, dirName), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + testFileName3 := dirName + string(os.PathSeparator) + "dest-" + fileName + ".txt" + s.createFile(testFileName3, text, c) + + syncArgs := []string{CloudURLToString(bucketName, dirName), dirName} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, ""), dirName, "-f", "--include", dirName + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + bDelete := true + backupDir := "test-backup-dir" + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + "delete": &bDelete, + "backupDir": &backupDir, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg file exist + _, err = os.Stat(testFileName1) + c.Assert(err, IsNil) + + //check, jpg file exist + _, err = os.Stat(testFileName2) + c.Assert(err, IsNil) + + //check, file not exist + _, err = os.Stat(testFileName3) + c.Assert(err, NotNil) + + //check, backup file exist + backupFile := backupDir + string(os.PathSeparator) + "dest-" + fileName + ".txt" + _, err = os.Stat(backupFile) + c.Assert(err, IsNil) + + os.RemoveAll(dirName) + os.RemoveAll(backupDir) + s.removeBucket(bucketName, true, c) +} + func (s *OssutilCommandSuite) TestSyncCopyIncludeFilter(c *C) { bucketName := bucketNamePrefix + randLowStr(10) s.putBucket(bucketName, c) @@ -999,6 +1134,534 @@ func (s *OssutilCommandSuite) TestSyncCopyIncludeFilter(c *C) { s.removeBucket(bucketName, true, c) } +func (s *OssutilCommandSuite) TestSyncCopyIncludeDirFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + prefix1 := "prefix1" + prefix2 := "prefix2" + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, prefix1), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + syncArgs := []string{CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2)} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2), "-f", "--include", prefix1 + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, prefix1+"/"+object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, prefix1+"/"+object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, txt object exist + objectStat = s.getStat(bucketName, prefix2+"/"+object1, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, prefix2+"/"+object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncUploadExcludeFilterSuccess(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // sync dir to oss + syncArgs := []string{dirName, CloudURLToString(bucketName, "")} + cmdline := []string{"ossutil", "sync", dirName, CloudURLToString(bucketName, ""), "-f", "--exclude", "*.txt"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, get stat + objectStat := s.getStat(bucketName, object2, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + _, err = s.rawGetStat(bucketName, object1) + c.Assert(err, NotNil) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncUploadExcludeDirFilterSuccess(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // sync dir to oss + syncArgs := []string{dirName, CloudURLToString(bucketName, "")} + cmdline := []string{"ossutil", "sync", dirName, CloudURLToString(bucketName, ""), "-f", "--exclude", dirName + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, get stat + _, err = s.rawGetStat(bucketName, object1) + c.Assert(err, NotNil) + + _, err = s.rawGetStat(bucketName, object2) + c.Assert(err, NotNil) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncDownloadExcludeFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, ""), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + testFileName3 := dirName + string(os.PathSeparator) + "dest-" + fileName + ".txt" + s.createFile(testFileName3, text, c) + + syncArgs := []string{CloudURLToString(bucketName, ""), dirName} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, ""), dirName, "-f", "--exclude", "*.txt"} + str := "" + cpDir := CheckpointDir + bForce := true + bDelete := true + backupDir := "test-backup-dir" + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + "delete": &bDelete, + "backupDir": &backupDir, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg file exist + _, err = os.Stat(testFileName1) + c.Assert(err, NotNil) + + //check, jpg file exist + _, err = os.Stat(testFileName2) + c.Assert(err, IsNil) + + //check, file not exist + _, err = os.Stat(testFileName3) + c.Assert(err, NotNil) + + //check, backup file exist + backupFile := backupDir + string(os.PathSeparator) + "dest-" + fileName + ".txt" + _, err = os.Stat(backupFile) + c.Assert(err, IsNil) + + os.RemoveAll(dirName) + os.RemoveAll(backupDir) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncDownloadExcludeDirFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := dirName + "/" + fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := dirName + "/" + fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, dirName), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + testFileName3 := dirName + string(os.PathSeparator) + "dest-" + fileName + ".txt" + s.createFile(testFileName3, text, c) + + syncArgs := []string{CloudURLToString(bucketName, dirName), dirName} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, ""), dirName, "-f", "--exclude", dirName + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + bDelete := true + backupDir := "test-backup-dir" + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + "delete": &bDelete, + "backupDir": &backupDir, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg file exist + _, err = os.Stat(testFileName1) + c.Assert(err, NotNil) + + //check, jpg file exist + _, err = os.Stat(testFileName2) + c.Assert(err, NotNil) + + //check, file not exist + _, err = os.Stat(testFileName3) + c.Assert(err, NotNil) + + //check, backup file exist + backupFile := backupDir + string(os.PathSeparator) + "dest-" + fileName + ".txt" + _, err = os.Stat(backupFile) + c.Assert(err, IsNil) + + os.RemoveAll(dirName) + os.RemoveAll(backupDir) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncCopyExcludeFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + prefix1 := "prefix1" + prefix2 := "prefix2" + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, prefix1), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + syncArgs := []string{CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2)} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2), "-f", "--exclude", "*.txt"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, prefix1+"/"+object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, prefix1+"/"+object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, txt object exist + objectStat = s.getStat(bucketName, prefix2+"/"+object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + _, err = s.rawGetStat(bucketName, prefix2+"/"+object1) + c.Assert(err, NotNil) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncCopyExcludeDirFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + text := randLowStr(100) + // dir + dirName := "testdir1-" + randLowStr(3) + fileName := "testfile-" + randLowStr(5) + + testFileName1 := dirName + string(os.PathSeparator) + fileName + ".txt" + object1 := fileName + ".txt" + testFileName2 := dirName + string(os.PathSeparator) + fileName + ".jpg" + object2 := fileName + ".jpg" + + err := os.MkdirAll(dirName, 0755) + s.createFile(testFileName1, text, c) + s.createFile(testFileName2, text, c) + + prefix1 := "prefix1" + prefix2 := "prefix2" + + // raw cp to oss + _, err = s.rawCP(dirName, CloudURLToString(bucketName, prefix1), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + + syncArgs := []string{CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2)} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, prefix1), CloudURLToString(bucketName, prefix2), "-f", "--exclude", prefix1 + "/*"} + str := "" + cpDir := CheckpointDir + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + } + + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + //check, txt object exist + objectStat := s.getStat(bucketName, prefix1+"/"+object1, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, jpg object exist + objectStat = s.getStat(bucketName, prefix1+"/"+object2, c) + etag = objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + + //check, txt object not exist + _, err = s.rawGetStat(bucketName, prefix2+"/"+object1) + c.Assert(err, NotNil) + _, err = s.rawGetStat(bucketName, prefix2+"/"+object2) + c.Assert(err, NotNil) + + os.RemoveAll(dirName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestSyncCopyExcludeDirAndIncludeDirFilter(c *C) { + bucketName := bucketNamePrefix + randLowStr(10) + s.putBucket(bucketName, c) + + destBucketName := bucketName + "-dest" + s.putBucket(destBucketName, c) + // dir1 + dirName1 := "testdir1-" + randLowStr(3) + subDirName1 := "subdir1-" + randLowStr(4) + filePrefix1 := "prefix1" + fileNameList1 := s.prepareTestFiles(dirName1, subDirName1, filePrefix1, "", 3, c) + + // upload dir2 without prefix on destBucket + _, err := s.rawCP(dirName1, CloudURLToString(bucketName, dirName1), true, true, false, DefaultBigFileThreshold, CheckpointDir) + c.Assert(err, IsNil) + // sync to dest bucket + syncArgs := []string{CloudURLToString(bucketName, ""), CloudURLToString(destBucketName, "")} + cmdline := []string{"ossutil", "sync", CloudURLToString(bucketName, ""), CloudURLToString(destBucketName, ""), "-f", "--include", dirName1 + "/*", "--exclude", "*/" + subDirName1 + "/*"} + str := "" + cpDir := CheckpointDir + bDelete := true + bForce := true + routines := strconv.Itoa(Routines) + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "force": &bForce, + "delete": &bDelete, + } + os.Args = cmdline + _, err = cm.RunCommand("sync", syncArgs, options) + os.Args = []string{} + c.Assert(err, IsNil) + + // check dest bucketname, fileNameList1,not exist, are deleted + for _, v := range fileNameList1 { + object := dirName1 + "/" + strings.Replace(v, string(os.PathSeparator), "/", -1) + if strings.Contains(v, subDirName1) { + _, err = s.rawGetStat(destBucketName, object) + c.Assert(err, NotNil) + } else { + objectStat := s.getStat(destBucketName, object, c) + etag := objectStat["Etag"] + c.Assert(len(etag) > 0, Equals, true) + } + } + + os.RemoveAll(dirName1) + s.removeBucket(bucketName, true, c) + s.removeBucket(destBucketName, true, c) +} + func (s *OssutilCommandSuite) TestSyncWithPayerSucess(c *C) { bucketName := payerBucket diff --git a/lib/util.go b/lib/util.go index b52a83ee..5edda383 100644 --- a/lib/util.go +++ b/lib/util.go @@ -219,12 +219,12 @@ func getFilter(cmdline []string) (bool, []filterOptionType) { filter.pattern = strings.Replace(strArg, "[!", "[^", -1) dir, _ := filepath.Split(filter.pattern) if dir != "" { - return false, filters + filter.isDir = true + filter.pattern = strArg } filters = append(filters, filter) } } - return true, filters } @@ -242,21 +242,35 @@ func containsInStrsSlice(vs []string, t string) bool { } func filterSingleStr(v, p string, include bool) bool { - _, name := filepath.Split(v) - res, _ := filepath.Match(p, name) - + res := filterObject(p, v, false) if include { return res } else { return !res } } - +func filterSingleDir(v, p string, include bool) bool { + res := filterObject(p, v, true) + if include { + return res + } else { + return !res + } +} +func filterObject(p, v string, isDir bool) bool { + res := false + if isDir { + res, _ = filepath.Match(p, v) + } else { + _, name := filepath.Split(v) + res, _ = filepath.Match(p, name) + } + return res +} func filterStrsWithInclude(vs []string, p string) []string { vsf := make([]string, 0) for _, v := range vs { - _, name := filepath.Split(v) - res, _ := filepath.Match(p, name) + res := filterSingleStr(v, p, true) if res { vsf = append(vsf, v) } @@ -264,12 +278,32 @@ func filterStrsWithInclude(vs []string, p string) []string { } return vsf } +func filterDirsWithInclude(vs []string, p string) []string { + vsf := make([]string, 0) + for _, v := range vs { + res := filterSingleDir(v, p, true) + if res { + vsf = append(vsf, v) + } + } + return vsf +} func filterStrsWithExclude(vs []string, p string) []string { vsf := make([]string, 0) for _, v := range vs { - _, name := filepath.Split(v) - res, _ := filepath.Match(p, name) + res := filterObject(p, v, false) + if !res { + vsf = append(vsf, v) + } + + } + return vsf +} +func filterDirsWithExclude(vs []string, p string) []string { + vsf := make([]string, 0) + for _, v := range vs { + res := filterObject(p, v, true) if !res { vsf = append(vsf, v) } @@ -285,16 +319,33 @@ func matchFiltersForStr(str string, filters []filterOptionType) bool { var res bool if filters[0].name == IncludePrompt { - res = filterSingleStr(str, filters[0].pattern, true) + if filters[0].isDir { + res = filterSingleDir(str, filters[0].pattern, true) + } else { + res = filterSingleStr(str, filters[0].pattern, true) + } } else { - res = filterSingleStr(str, filters[0].pattern, false) + if filters[0].isDir { + res = filterSingleDir(str, filters[0].pattern, false) + } else { + res = filterSingleStr(str, filters[0].pattern, false) + } + } for _, filter := range filters[1:] { if filter.name == IncludePrompt { - res = res || filterSingleStr(str, filter.pattern, true) + if filter.isDir { + res = res || filterSingleDir(str, filter.pattern, true) + } else { + res = res || filterSingleStr(str, filter.pattern, true) + } } else { - res = res && filterSingleStr(str, filter.pattern, false) + if filter.isDir { + res = res && filterSingleDir(str, filter.pattern, false) + } else { + res = res && filterSingleStr(str, filter.pattern, false) + } } } @@ -324,16 +375,32 @@ func matchFiltersForStrsInArray(strs []string, filters []filterOptionType) []str vsf := make([]string, 0) if filters[0].name == IncludePrompt { - vsf = append(vsf, filterStrsWithInclude(strs, filters[0].pattern)...) + if filters[0].isDir { + vsf = append(vsf, filterDirsWithInclude(strs, filters[0].pattern)...) + } else { + vsf = append(vsf, filterStrsWithInclude(strs, filters[0].pattern)...) + } } else { - vsf = append(vsf, filterStrsWithExclude(strs, filters[0].pattern)...) + if filters[0].isDir { + vsf = append(vsf, filterDirsWithExclude(strs, filters[0].pattern)...) + } else { + vsf = append(vsf, filterStrsWithExclude(strs, filters[0].pattern)...) + } } for _, filter := range filters[1:] { if filter.name == IncludePrompt { - vsf = append(vsf, filterStrsWithInclude(strs, filter.pattern)...) + if filter.isDir { + vsf = append(vsf, filterDirsWithInclude(strs, filter.pattern)...) + } else { + vsf = append(vsf, filterStrsWithInclude(strs, filter.pattern)...) + } } else { - vsf = filterStrsWithExclude(vsf, filter.pattern) + if filter.isDir { + vsf = filterDirsWithExclude(vsf, filter.pattern) + } else { + vsf = filterStrsWithExclude(vsf, filter.pattern) + } } } @@ -341,8 +408,8 @@ func matchFiltersForStrsInArray(strs []string, filters []filterOptionType) []str } // Following for files - func doesSingleFileMatchPatterns(filename string, filters []filterOptionType) bool { + filename = toLinux(filename) if len(filters) == 0 { return true } @@ -356,6 +423,26 @@ func doesSingleFileMatchPatterns(filename string, filters []filterOptionType) bo return false } +// toLinux windows "\\" turn to "/" +func toLinux(filename string) string { + if strings.Contains(filename, "\\") { + filename = strings.ReplaceAll(filename, "\\", "/") + } + return filename +} + +// getFullFilePath from fileInfoType get the full file name +func getFullFilePath(v fileInfoType) string { + fullFilePath := v.dir + v.filePath + fullFilePath = toLinux(fullFilePath) + return fullFilePath +} + +// getFullObjectKey from objectInfoType get the full object name +func getFullObjectKey(v objectInfoType) string { + objectKey := v.prefix + v.relativeKey + return objectKey +} func containsInFileSlice(vs []fileInfoType, t fileInfoType) bool { if len(vs) == 0 { return false @@ -372,9 +459,20 @@ func containsInFileSlice(vs []fileInfoType, t fileInfoType) bool { func filterFilesWithInclude(vs []fileInfoType, p string) []fileInfoType { vsf := make([]fileInfoType, 0) for _, v := range vs { - _, filename := filepath.Split(v.filePath) - res, _ := filepath.Match(p, filename) + res := filterObject(p, v.filePath, false) + if res { + vsf = append(vsf, v) + } + + } + return vsf +} +func filterFilesDirWithInclude(vs []fileInfoType, p string) []fileInfoType { + vsf := make([]fileInfoType, 0) + for _, v := range vs { + name := getFullFilePath(v) + res := filterObject(p, name, true) if res { vsf = append(vsf, v) } @@ -386,8 +484,19 @@ func filterFilesWithInclude(vs []fileInfoType, p string) []fileInfoType { func filterFilesWithExclude(vs []fileInfoType, p string) []fileInfoType { vsf := make([]fileInfoType, 0) for _, v := range vs { - _, filename := filepath.Split(v.filePath) - res, _ := filepath.Match(p, filename) + res := filterObject(p, v.filePath, false) + if !res { + vsf = append(vsf, v) + } + + } + return vsf +} +func filterFilesDirWithExclude(vs []fileInfoType, p string) []fileInfoType { + vsf := make([]fileInfoType, 0) + for _, v := range vs { + name := getFullFilePath(v) + res := filterObject(p, name, true) if !res { vsf = append(vsf, v) } @@ -418,7 +527,17 @@ func filterObjectsWithInclude(vs []objectInfoType, p string) []objectInfoType { vsf := make([]objectInfoType, 0) for _, v := range vs { _, key := filepath.Split(v.relativeKey) - //_, key := filepath.Split(v.key) + res, _ := filepath.Match(p, key) + if res { + vsf = append(vsf, v) + } + } + return vsf +} +func filterObjectsWithIncludeDir(vs []objectInfoType, p string) []objectInfoType { + vsf := make([]objectInfoType, 0) + for _, v := range vs { + key := getFullObjectKey(v) res, _ := filepath.Match(p, key) if res { vsf = append(vsf, v) @@ -430,10 +549,19 @@ func filterObjectsWithInclude(vs []objectInfoType, p string) []objectInfoType { func filterObjectsWithExclude(vs []objectInfoType, p string) []objectInfoType { vsf := make([]objectInfoType, 0) for _, v := range vs { - _, key := filepath.Split(v.relativeKey) - //_, key := filepath.Split(v.key) - res, _ := filepath.Match(p, key) - if !res { + res := filterSingleStr(v.relativeKey, p, false) + if res { + vsf = append(vsf, v) + } + } + return vsf +} +func filterObjectsWithExcludeDir(vs []objectInfoType, p string) []objectInfoType { + vsf := make([]objectInfoType, 0) + for _, v := range vs { + key := getFullObjectKey(v) + res := filterSingleDir(key, p, false) + if res { vsf = append(vsf, v) } } @@ -449,13 +577,27 @@ func matchFiltersForObjects(objects []objectInfoType, filters []filterOptionType for i, filter := range filters { if filter.name == IncludePrompt { - vsf = append(vsf, filterObjectsWithInclude(objects, filter.pattern)...) + if filter.isDir { + vsf = append(vsf, filterObjectsWithIncludeDir(objects, filter.pattern)...) + } else { + vsf = append(vsf, filterObjectsWithInclude(objects, filter.pattern)...) + } + } else { - if i == 0 { - vsf = append(vsf, filterObjectsWithExclude(objects, filter.pattern)...) + if filter.isDir { + if i == 0 { + vsf = append(vsf, filterObjectsWithExcludeDir(objects, filter.pattern)...) + } else { + vsf = filterObjectsWithExcludeDir(vsf, filter.pattern) + } } else { - vsf = filterObjectsWithExclude(vsf, filter.pattern) + if i == 0 { + vsf = append(vsf, filterObjectsWithExclude(objects, filter.pattern)...) + } else { + vsf = filterObjectsWithExclude(vsf, filter.pattern) + } } + } } @@ -533,7 +675,7 @@ func currentHomeDir() string { return homeDir } -func getCurrentDirFileListCommon(dpath string, chFiles chan<- fileInfoType, filters []filterOptionType) error { +func getCurrentDirFileListCommon(dpath string, chFiles chan<- fileInfoType, filters []filterOptionType, dest bool) error { if !strings.HasSuffix(dpath, string(os.PathSeparator)) { dpath += string(os.PathSeparator) } @@ -550,20 +692,24 @@ func getCurrentDirFileListCommon(dpath string, chFiles chan<- fileInfoType, filt // for symlink continue } - - if doesSingleFileMatchPatterns(fileInfo.Name(), filters) { + if dest { chFiles <- fileInfoType{fileInfo.Name(), dpath} + } else { + if doesSingleFileMatchPatterns(fileInfo.Name(), filters) { + chFiles <- fileInfoType{fileInfo.Name(), dpath} + } } + } } return nil } func getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir bool, disableAllSymlink bool, - enableSymlinkDir bool, filters []filterOptionType) error { + enableSymlinkDir bool, filters []filterOptionType, dest bool) error { defer close(chFiles) if onlyCurrentDir { - return getCurrentDirFileListCommon(dpath, chFiles, filters) + return getCurrentDirFileListCommon(dpath, chFiles, filters, dest) } name := dpath @@ -583,11 +729,16 @@ func getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir if f.IsDir() { if fpath != dpath { - if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { - chFiles <- fileInfoType{fileName, name} - } else { - chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} + if !dest { + if doesSingleFileMatchPatterns(fpath, filters) { + if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { + chFiles <- fileInfoType{fileName, name} + } else { + chFiles <- fileInfoType{fileName + string(os.PathSeparator), name} + } + } } + } return nil } @@ -614,9 +765,12 @@ func getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir return nil } } - - if doesSingleFileMatchPatterns(fileName, filters) { + if dest { chFiles <- fileInfoType{fileName, name} + } else { + if doesSingleFileMatchPatterns(fpath, filters) { + chFiles <- fileInfoType{fileName, name} + } } return nil } @@ -639,7 +793,7 @@ func getFileListCommon(dpath string, chFiles chan<- fileInfoType, onlyCurrentDir } func getObjectListCommon(bucket *oss.Bucket, cloudURL CloudURL, chObjects chan<- objectInfoType, - onlyCurrentDir bool, filters []filterOptionType, payerOptions []oss.Option) error { + onlyCurrentDir bool, filters []filterOptionType, payerOptions []oss.Option, dest bool) error { defer close(chObjects) pre := oss.Prefix(cloudURL.object) marker := oss.Marker("") @@ -667,8 +821,7 @@ func getObjectListCommon(bucket *oss.Bucket, cloudURL CloudURL, chObjects chan<- prefix = object.Key[:index+1] relativeKey = object.Key[index+1:] } - - if doesSingleObjectMatchPatterns(object.Key, filters) { + if dest { if strings.ToLower(object.Type) == "symlink" { props, err := bucket.GetObjectDetailedMeta(object.Key, payerOptions...) if err != nil { @@ -684,6 +837,24 @@ func getObjectListCommon(bucket *oss.Bucket, cloudURL CloudURL, chObjects chan<- object.Size = size } chObjects <- objectInfoType{prefix, relativeKey, int64(object.Size), object.LastModified} + } else { + if doesSingleObjectMatchPatterns(object.Key, filters) { + if strings.ToLower(object.Type) == "symlink" { + props, err := bucket.GetObjectDetailedMeta(object.Key, payerOptions...) + if err != nil { + LogError("ossGetObjectStatRetry error info:%s\n", err.Error()) + return err + } + size, err := strconv.ParseInt(props.Get(oss.HTTPHeaderContentLength), 10, 64) + if err != nil { + LogError("strconv.ParseInt error info:%s\n", err.Error()) + return err + + } + object.Size = size + } + chObjects <- objectInfoType{prefix, relativeKey, int64(object.Size), object.LastModified} + } } }