diff --git a/lib/command_test.go b/lib/command_test.go
index a4043339..9a3197e1 100644
--- a/lib/command_test.go
+++ b/lib/command_test.go
@@ -58,6 +58,7 @@ var (
downloadFileName = "ossutil_test.download_file" + randStr(5)
downloadDir = "ossutil_test.download_dir" + randStr(5)
inputFileName = "ossutil_test.input_file" + randStr(5)
+ objectFileName = "ossutil_test.object_file" + randStr(5)
content = "abc"
cm = CommandManager{}
out = os.Stdout
@@ -1415,6 +1416,16 @@ func (s *OssutilCommandSuite) initSetMetaWithArgs(args []string, cmdline string,
cmdline = cmdline[0:pos] + cmdline[pos+len("--delete"):]
}
+ var disableIgnoreError bool
+ if pos := strings.Index(cmdline, "--disable-ignore-error"); pos != -1 {
+ disableIgnoreError = true
+ cmdline = cmdline[0:pos] + cmdline[pos+len("--disable-ignore-error"):]
+ }
+
+ objectFile := ""
+ snapshotPath := ""
+ objectFile, snapshotPath, cmdline = s.handleObjectFileSnapshot(cmdline)
+
parameter := strings.Split(cmdline, "-")
r := false
f := false
@@ -1426,17 +1437,20 @@ func (s *OssutilCommandSuite) initSetMetaWithArgs(args []string, cmdline string,
str := ""
routines := strconv.Itoa(Routines)
options := OptionMapType{
- "endpoint": &str,
- "accessKeyID": &str,
- "accessKeySecret": &str,
- "stsToken": &str,
- "configFile": &configFile,
- "update": &update,
- "delete": &delete,
- "recursive": &r,
- "force": &f,
- "routines": &routines,
- "encodingType": &encodingType,
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "update": &update,
+ "delete": &delete,
+ "recursive": &r,
+ "force": &f,
+ "routines": &routines,
+ "encodingType": &encodingType,
+ "objectFile": &objectFile,
+ "snapshotPath": &snapshotPath,
+ "disableIgnoreError": &disableIgnoreError,
}
err := setMetaCommand.Init(args, options)
return err
@@ -1582,6 +1596,33 @@ func (s *OssutilCommandSuite) getFileList(dpath string) ([]string, error) {
return fileList, err
}
+func (s *OssutilCommandSuite) handleObjectFileSnapshot(cmdline string) (objectFile, snapshotPath, newCmdline string) {
+ cmds := strings.Split(cmdline, " ")
+ for i := 0; i < len(cmds); i++ {
+ if cmds[i] == "--object-file" && !strings.HasPrefix(cmds[i+1], "-") {
+ objectFile = cmds[i+1]
+ } else {
+ continue
+ }
+ cmds = append(cmds[:i], cmds[i+2:]...)
+ }
+ for i := 0; i < len(cmds); i++ {
+ if cmds[i] == "--snapshot-path" && !strings.HasPrefix(cmds[i+1], "-") {
+ snapshotPath = cmds[i+1]
+ } else {
+ continue
+ }
+ cmds = append(cmds[:i], cmds[i+2:]...)
+ }
+
+ // remake cmdline
+ cmdline = ""
+ for i := 0; i < len(cmds); i++ {
+ cmdline = cmdline + " " + cmds[i]
+ }
+ return objectFile, snapshotPath, cmdline
+}
+
func (s *OssutilCommandSuite) initRestoreObject(args []string, cmdline string, outputDir string) error {
encodingType := ""
if pos := strings.Index(cmdline, "--encoding-type url"); pos != -1 {
@@ -1589,6 +1630,16 @@ func (s *OssutilCommandSuite) initRestoreObject(args []string, cmdline string, o
cmdline = cmdline[0:pos] + cmdline[pos+len("--encoding-type url"):]
}
+ var disableIgnoreError bool
+ if pos := strings.Index(cmdline, "--disable-ignore-error"); pos != -1 {
+ disableIgnoreError = true
+ cmdline = cmdline[0:pos] + cmdline[pos+len("--disable-ignore-error"):]
+ }
+
+ objectFile := ""
+ snapshotPath := ""
+ objectFile, snapshotPath, cmdline = s.handleObjectFileSnapshot(cmdline)
+
parameter := strings.Split(cmdline, "-")
r := false
f := false
@@ -1600,16 +1651,19 @@ func (s *OssutilCommandSuite) initRestoreObject(args []string, cmdline string, o
str := ""
routines := strconv.Itoa(Routines)
options := OptionMapType{
- "endpoint": &str,
- "accessKeyID": &str,
- "accessKeySecret": &str,
- "stsToken": &str,
- "configFile": &configFile,
- "recursive": &r,
- "force": &f,
- "encodingType": &encodingType,
- "routines": &routines,
- "outputDir": &outputDir,
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "recursive": &r,
+ "force": &f,
+ "encodingType": &encodingType,
+ "routines": &routines,
+ "outputDir": &outputDir,
+ "objectFile": &objectFile,
+ "snapshotPath": &snapshotPath,
+ "disableIgnoreError": &disableIgnoreError,
}
err := restoreCommand.Init(args, options)
return err
diff --git a/lib/const.go b/lib/const.go
index a61ecd14..0c62e407 100644
--- a/lib/const.go
+++ b/lib/const.go
@@ -99,6 +99,7 @@ const (
OptionSkipVerfiyCert = "skipVerifyCert"
OptionItem = "item"
OptionUserAgent = "userAgent"
+ OptionObjectFile = "objectFile"
)
// the elements show in stat object
diff --git a/lib/monitor.go b/lib/monitor.go
index 29d91e7c..8f33cacf 100644
--- a/lib/monitor.go
+++ b/lib/monitor.go
@@ -36,6 +36,7 @@ type Monitorer interface {
type MonitorSnap struct {
okNum int64
errNum int64
+ skipNum int64
dealNum int64
}
@@ -49,6 +50,7 @@ type Monitor struct {
totalNum int64
okNum int64
errNum int64
+ skipNum int64
seekAheadError error
seekAheadEnd bool
finish bool
@@ -62,6 +64,7 @@ func (m *Monitor) init(opStr string) {
m.seekAheadError = nil
m.okNum = 0
m.errNum = 0
+ m.skipNum = 0
m.finish = false
}
@@ -90,6 +93,7 @@ func (m *Monitor) getSnapshot() *MonitorSnap {
var snap MonitorSnap
snap.okNum = m.okNum
snap.errNum = m.errNum
+ snap.skipNum = m.skipNum
snap.dealNum = snap.okNum + snap.errNum
return &snap
}
@@ -141,24 +145,24 @@ func (m *Monitor) getWholeFinishBar() string {
snap := m.getSnapshot()
if m.seekAheadEnd && m.seekAheadError == nil {
if snap.errNum == 0 {
- return getClearStr(fmt.Sprintf("Succeed: Total %d objects. %s %d objects.\n", m.totalNum, m.opStr, snap.okNum))
+ return getClearStr(fmt.Sprintf("Succeed: Total %d objects. %s %d objects(skip %d objects).\n", m.totalNum, m.opStr, snap.okNum, snap.skipNum))
}
- return getClearStr(fmt.Sprintf("FinishWithError: Total %d objects. %s %d objects, Error %d objects.\n", m.totalNum, m.opStr, snap.okNum, snap.errNum))
+ return getClearStr(fmt.Sprintf("FinishWithError: Total %d objects. %s %d objects(skip %d objects), Error %d objects.\n", m.totalNum, m.opStr, snap.okNum, snap.skipNum, snap.errNum))
}
scanNum := max(m.totalNum, snap.dealNum)
if snap.errNum == 0 {
- return getClearStr(fmt.Sprintf("Succeed: Total %d objects. %s %d objects.\n", scanNum, m.opStr, snap.okNum))
+ return getClearStr(fmt.Sprintf("Succeed: Total %d objects. %s %d objects(skip %d objects).\n", scanNum, m.opStr, snap.okNum, snap.skipNum))
}
- return getClearStr(fmt.Sprintf("FinishWithError: Scanned %d objects. %s %d objects, Error %d objects.\n", scanNum, m.opStr, snap.okNum, snap.errNum))
+ return getClearStr(fmt.Sprintf("FinishWithError: Scanned %d objects. %s %d objects(skip %d objects), Error %d objects.\n", scanNum, m.opStr, snap.okNum, snap.skipNum, snap.errNum))
}
func (m *Monitor) getDefeatBar() string {
snap := m.getSnapshot()
if m.seekAheadEnd && m.seekAheadError == nil {
- return getClearStr(fmt.Sprintf("Total %d objects. %s %d objects, when error happens.\n", m.totalNum, m.opStr, snap.okNum))
+ return getClearStr(fmt.Sprintf("Total %d objects. %s %d objects(skip %d objects), when error happens.\n", m.totalNum, m.opStr, snap.okNum, snap.skipNum))
}
scanNum := max(m.totalNum, snap.dealNum)
- return getClearStr(fmt.Sprintf("Scanned %d objects. %s %d objects, when error happens.\n", scanNum, m.opStr, snap.okNum))
+ return getClearStr(fmt.Sprintf("Scanned %d objects. %s %d objects(skip %d objects), when error happens.\n", scanNum, m.opStr, snap.okNum, snap.skipNum))
}
// For rm
diff --git a/lib/option.go b/lib/option.go
index 56e528af..5fa7afb7 100644
--- a/lib/option.go
+++ b/lib/option.go
@@ -265,6 +265,9 @@ var OptionMap = map[string]Option{
OptionUserAgent: Option{"", "--ua", "", OptionTypeString, "", "",
"指定http请求中的user agent, 会在缺省值后面加上指定值",
"Specify the user agent in the http request, and the specified value will be added after the default value"},
+ OptionObjectFile: Option{"", "--object-file", "", OptionTypeString, "", "",
+ "表示所有待处理的objects，取值为一个存在的文件路径",
+ "Specify all the objects that need to be operated, and the specified value should be a exists file path"},
}
func (T *Option) getHelp(language string) string {
diff --git a/lib/restore.go b/lib/restore.go
index fabfec5d..dd57d38b 100644
--- a/lib/restore.go
+++ b/lib/restore.go
@@ -1,17 +1,22 @@
package lib
import (
+ "bufio"
"fmt"
+ oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "github.com/syndtr/goleveldb/leveldb"
"io/ioutil"
"os"
"strings"
-
- oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "sync/atomic"
+ "time"
)
type batchOptionType struct {
- ctnu bool
- reporter *Reporter
+ ctnu bool
+ reporter *Reporter
+ snapshotPath string
+ snapshotldb *leveldb.DB
}
var specChineseRestore = SpecText{
@@ -21,7 +26,7 @@ var specChineseRestore = SpecText{
paramText: "cloud_url [local_xml_file] [options]",
syntaxText: `
- ossutil restore cloud_url [local_xml_file] [--encoding-type url] [-r] [-f] [--output-dir=odir] [--version-id versionId] [--payer requester] [-c file]
+ ossutil restore cloud_url [local_xml_file] [--encoding-type url] [-r] [-f] [--output-dir=odir] [--version-id versionId] [--payer requester] [-c file] [--object-file file] [--snapshot-path dir] [--disable-ignore-error]
`,
detailHelpText: `
@@ -64,6 +69,18 @@ var specChineseRestore = SpecText{
Bulk
+
+ 3) ossutil restore oss://bucket --object-file file [--snapshot-path dir] [--disable-ignore-error] [--output-dir=odir] [local_xml_file]
+ 该用法可批量恢复多个冷冻状态的objects为可读状态，与-r的区别在于，-r匹配所有符合
+ 指定前缀的objects，而--object-file指定对某些object进行处理。ossutil会读取文件中
+ 的objects，恢复它们为可读状态。当一个object操作出现错误时，会将出错object的错误信息
+ 记录到report文件，并继续操作其他object，成功操作的object信息将不会被记录到report
+ 文件中（更多信息见cp命令的帮助）。如果--force选项被指定，则不会进行询问提示。
+
+ 上面的file是本地文件, 里面罗列了所有需要处理的object，之间以"\n"隔开, 举例如下
+ object1
+ object2
+ object3
`,
sampleText: `
@@ -73,6 +90,8 @@ var specChineseRestore = SpecText{
4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
5) ossutil restore oss://bucket-restore/object-store --payer requester
6) ossutil restore oss://bucket-restore/object-prefix -r -f local_xml_file
+ 7) ossutil restore oss://bucket-restore --object-file file -f local_xml_file
+ 8) ossutil restore oss://bucket-restore --object-file file --snapshot-path dir -f local_xml_file
`,
}
@@ -83,7 +102,7 @@ var specEnglishRestore = SpecText{
paramText: "cloud_url [local_xml_file] [options]",
syntaxText: `
- ossutil restore cloud_url [local_xml_file] [--encoding-type url] [-r] [-f] [--output-dir=odir] [--version-id versionId] [--payer requester] [-c file]
+ ossutil restore cloud_url [local_xml_file] [--encoding-type url] [-r] [-f] [--output-dir=odir] [--version-id versionId] [--payer requester] [-c file] [--object-file file] [--snapshot-path dir] [--disable-ignore-error]
`,
detailHelpText: `
@@ -130,6 +149,20 @@ Usage:
Bulk
+
+ 3) ossutil restore oss://bucket --object-file file [--snapshot-path dir] [--disable-ignore-error] [--output-dir=odir] [local_xml_file]
+ The usage restore the objects with the specified prefix and in frozen state to readable status.
+ The difference with -r is that -r matches all objects that match the specified prefix, while
+ --object-file specifies some objects to be restored. Ossutil reads the objects in the file and
+ restores them to a readable state. When an error occurs when restore an object, ossutil will
+ record the error message to report file, and ossutil will continue to attempt to set acl on the
+ remaining objects(more information see help of cp command). If --force option is specified,
+ ossutil will not show prompt question.
+
+ The file is a local file, which lists all the objects to be restored, separated by "\n". For example:
+ object1
+ object2
+ object3
`,
sampleText: `
@@ -139,6 +172,8 @@ Usage:
4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
5) ossutil restore oss://bucket-restore/object-store --payer requester
6) ossutil restore oss://bucket-restore/object-prefix -r -f local_xml_file
+ 7) ossutil restore oss://bucket-restore --object-file file -f local_xml_file
+ 8) ossutil restore oss://bucket-restore --object-file file --snapshot-path dir -f local_xml_file
`,
}
@@ -151,6 +186,8 @@ type RestoreCommand struct {
restoreConfig oss.RestoreConfiguration
hasConfig bool
configXml string
+ hasObjFile bool
+ objFilePath string
}
var restoreCommand = RestoreCommand{
@@ -191,6 +228,9 @@ var restoreCommand = RestoreCommand{
OptionSTSRegion,
OptionSkipVerfiyCert,
OptionUserAgent,
+ OptionObjectFile,
+ OptionSnapshotPath,
+ OptionDisableIgnoreError,
},
},
}
@@ -211,11 +251,13 @@ func (rc *RestoreCommand) Init(args []string, options OptionMapType) error {
// RunCommand simulate inheritance, and polymorphism
func (rc *RestoreCommand) RunCommand() error {
- rc.monitor.init("Restored")
-
+ rc.monitor.init("Restore")
encodingType, _ := GetString(OptionEncodingType, rc.command.options)
recursive, _ := GetBool(OptionRecursion, rc.command.options)
versionid, _ := GetString(OptionVersionId, rc.command.options)
+ force, _ := GetBool(OptionForce, rc.command.options)
+ objFileXml, _ := GetString(OptionObjectFile, rc.command.options)
+ snapshotPath, _ := GetString(OptionSnapshotPath, rc.command.options)
payer, _ := GetString(OptionRequestPayer, rc.command.options)
if payer != "" {
@@ -225,78 +267,132 @@ func (rc *RestoreCommand) RunCommand() error {
rc.commonOptions = append(rc.commonOptions, oss.RequestPayer(oss.PayerType(payer)))
}
+ var err error
+ // load snapshot
+ rc.reOption.snapshotPath = snapshotPath
+ if rc.reOption.snapshotPath != "" {
+ if rc.reOption.snapshotldb, err = leveldb.OpenFile(rc.reOption.snapshotPath, nil); err != nil {
+ return fmt.Errorf("load snapshot error, reason: %s", err.Error())
+ }
+ defer rc.reOption.snapshotldb.Close()
+ }
+
cloudURL, err := CloudURLFromString(rc.command.args[0], encodingType)
if err != nil {
return err
}
-
- if err = rc.checkArgs(cloudURL, recursive, versionid); err != nil {
+ if err = rc.checkOptions(cloudURL, recursive, force, versionid, objFileXml); err != nil {
+ if err.Error() == "operation is canceled" {
+ return nil
+ }
+ return err
+ }
+ bucket, err := rc.command.ossBucket(cloudURL.bucket)
+ if err != nil {
return err
}
- if len(rc.command.args) == 2 {
- xmlFile := rc.command.args[1]
- fileInfo, err := os.Stat(xmlFile)
+ // parse restore config
+ if len(rc.command.args) > 1 {
+ restoreConf := rc.command.args[1]
+ err := rc.parseRestoreConf(restoreConf)
if err != nil {
return err
}
+ } else {
+ rc.hasConfig = false
+ }
- if fileInfo.IsDir() {
- return fmt.Errorf("%s is dir,not the expected file", xmlFile)
+ if objFileXml != "" {
+ // check objFileXml and parse it
+ if err := rc.checkObjectFile(objFileXml); err != nil {
+ return err
}
-
- if fileInfo.Size() == 0 {
- return fmt.Errorf("%s is empty file", xmlFile)
+ recursive = true
+ return rc.batchRestoreObjects(bucket, cloudURL, recursive)
+ } else {
+ if !recursive {
+ return rc.ossRestoreObject(bucket, cloudURL.object, versionid, false)
}
- // parsing the xml file
- file, err := os.Open(xmlFile)
- if err != nil {
- return err
+ return rc.batchRestoreObjects(bucket, cloudURL, recursive)
+ }
+}
+
+func (rc *RestoreCommand) checkOptions(cloudURL CloudURL, recursive, force bool, versionid, objectFile string) error {
+ if cloudURL.bucket == "" {
+ return fmt.Errorf("invalid cloud url: %s, miss bucket", cloudURL.urlStr)
+ }
+ if cloudURL.object == "" {
+ if !recursive && objectFile == "" {
+ return fmt.Errorf("restore object invalid cloud url: %s, object empty. Restore bucket is not supported, if you mean batch restore objects, please use --recursive or --object-file", rc.command.args[0])
}
- defer file.Close()
- text, err := ioutil.ReadAll(file)
- if err != nil {
- return err
+ } else {
+ if objectFile != "" {
+ return fmt.Errorf("the first arg of `ossutil restore` only support oss://bucket when set option --object-file.")
}
- rc.hasConfig = true
- rc.configXml = string(text)
}
- bucket, err := rc.command.ossBucket(cloudURL.bucket)
- if err != nil {
- return err
+ if (recursive && len(versionid) > 0) || (objectFile != "" && len(versionid) > 0) {
+ return fmt.Errorf("restore bucket dose not support the --version-id=%s argument.", versionid)
}
- if !recursive {
- return rc.ossRestoreObject(bucket, cloudURL.object, versionid)
+ if !force {
+ var val string
+ if recursive || objectFile != "" {
+ fmt.Printf("Do you really mean to recursivlly restore objects of %s(y or N)? ", rc.command.args[0])
+ if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
+ return fmt.Errorf("operation is canceled")
+ }
+ }
}
- return rc.batchRestoreObjects(bucket, cloudURL)
+
+ return nil
}
-func (rc *RestoreCommand) checkArgs(cloudURL CloudURL, recursive bool, versionid string) error {
- if cloudURL.bucket == "" {
- return fmt.Errorf("invalid cloud url: %s, miss bucket", rc.command.args[0])
+func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object, versionid string, batchOperate bool) error {
+ nowt := time.Now().Unix()
+ spath := ""
+ msg := "restore"
+
+ if batchOperate && rc.reOption.snapshotPath != "" {
+ spath = rc.formatSnapshotKey(bucket.BucketName, object, msg)
+ if skip := rc.skipRestore(spath); skip {
+ rc.updateSkip(1)
+ LogInfo("restore obj skip: %s\n", object)
+ return nil
+ }
}
- if !recursive && cloudURL.object == "" {
- return fmt.Errorf("restore object invalid cloud url: %s, object empty. Restore bucket is not supported, if you mean batch restore objects, please use --recursive", rc.command.args[0])
+
+ var options []oss.Option
+ if len(versionid) > 0 {
+ options = append(options, oss.VersionId(versionid))
}
- if recursive && len(versionid) > 0 {
- return fmt.Errorf("restore bucket dose not support the --version-id=%s argument.", versionid)
+ options = append(options, rc.commonOptions...)
+
+ err := rc.ossRestoreObjectRetry(bucket, object, options...)
+ if batchOperate && rc.reOption.snapshotPath != "" {
+ if err != nil {
+ _ = rc.updateSnapshot(err, spath, nowt)
+ return err
+ } else {
+ err = rc.updateSnapshot(err, spath, nowt)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ return err
}
+
return nil
}
-func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object string, versionid string) error {
+func (rc *RestoreCommand) ossRestoreObjectRetry(bucket *oss.Bucket, object string, options ...oss.Option) error {
+ var err error
retryTimes, _ := GetInt(OptionRetryTimes, rc.command.options)
- for i := 1; ; i++ {
- var options []oss.Option
- if len(versionid) > 0 {
- options = append(options, oss.VersionId(versionid))
- }
- options = append(options, rc.commonOptions...)
- var err error
+ for i := 1; ; i++ {
if rc.hasConfig {
err = bucket.RestoreObjectXML(object, rc.configXml, options...)
} else {
@@ -304,7 +400,7 @@ func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object string, ve
}
if err == nil {
- return err
+ return nil
}
switch err.(type) {
@@ -320,18 +416,12 @@ func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object string, ve
}
}
-func (rc *RestoreCommand) batchRestoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
- force, _ := GetBool(OptionForce, rc.command.options)
- if !force {
- var val string
- fmt.Printf("Do you really mean to recursivlly restore objects of %s(y or N)? ", rc.command.args[0])
- if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
- fmt.Println("operation is canceled.")
- return nil
- }
- }
-
+func (rc *RestoreCommand) batchRestoreObjects(bucket *oss.Bucket, cloudURL CloudURL, recursive bool) error {
rc.reOption.ctnu = true
+ if recursive || rc.hasObjFile {
+ disableIgnoreError, _ := GetBool(OptionDisableIgnoreError, rc.command.options)
+ rc.reOption.ctnu = !disableIgnoreError
+ }
outputDir, _ := GetString(OptionOutputDir, rc.command.options)
// init reporter
@@ -341,12 +431,14 @@ func (rc *RestoreCommand) batchRestoreObjects(bucket *oss.Bucket, cloudURL Cloud
}
defer rc.reOption.reporter.Clear()
+ if rc.hasObjFile {
+ return rc.restoreObjectsFromFile(bucket, cloudURL, rc.objFilePath)
+ }
return rc.restoreObjects(bucket, cloudURL)
}
func (rc *RestoreCommand) restoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
routines, _ := GetInt(OptionRoutines, rc.command.options)
-
chObjects := make(chan string, ChannelBuf)
chError := make(chan error, routines+1)
chListError := make(chan error, 1)
@@ -359,6 +451,69 @@ func (rc *RestoreCommand) restoreObjects(bucket *oss.Bucket, cloudURL CloudURL)
return rc.waitRoutinueComplete(chError, chListError, routines)
}
+func (rc *RestoreCommand) restoreObjectsFromFile(bucket *oss.Bucket, cloudURL CloudURL, objectFile string) error {
+ routines, _ := GetInt(OptionRoutines, rc.command.options)
+
+ chObjects := make(chan string, ChannelBuf)
+ chError := make(chan error, routines+1)
+ chListError := make(chan error, 1)
+ go rc.restoreStatistic(objectFile, &rc.monitor, []filterOptionType{}, rc.commonOptions...)
+ go rc.restoreProducer(objectFile, chObjects, chListError, []filterOptionType{}, rc.commonOptions...)
+ for i := 0; int64(i) < routines; i++ {
+ go rc.restoreConsumer(bucket, cloudURL, chObjects, chError)
+ }
+ return rc.waitRoutinueComplete(chError, chListError, routines)
+}
+
+func (rc *RestoreCommand) restoreStatistic(objectFile string, monitor Monitorer, filters []filterOptionType, options ...oss.Option) {
+ if monitor == nil {
+ return
+ }
+
+ file, err := os.Open(objectFile)
+ if err != nil {
+ monitor.setScanError(err)
+ return
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ object := scanner.Text()
+ object = strings.Trim(object, " ")
+ if object == "" {
+ monitor.setScanError(fmt.Errorf("object can't be '' in --object-file"))
+ return
+ }
+ monitor.updateScanNum(1)
+ }
+
+ monitor.setScanEnd()
+}
+
+func (rc *RestoreCommand) restoreProducer(objectFile string, chObjects chan<- string, chError chan<- error, filters []filterOptionType, options ...oss.Option) {
+ file, err := os.Open(objectFile)
+ if err != nil {
+ chError <- err
+ return
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ object := scanner.Text()
+ object = strings.Trim(object, " ")
+ if object == "" {
+ chError <- fmt.Errorf("object can't be '' in --object-file")
+ break
+ }
+ chObjects <- object
+ }
+
+ defer close(chObjects)
+ chError <- nil
+}
+
func (rc *RestoreCommand) restoreConsumer(bucket *oss.Bucket, cloudURL CloudURL, chObjects <-chan string, chError chan<- error) {
for object := range chObjects {
err := rc.restoreObjectWithReport(bucket, object)
@@ -375,7 +530,7 @@ func (rc *RestoreCommand) restoreConsumer(bucket *oss.Bucket, cloudURL CloudURL,
}
func (rc *RestoreCommand) restoreObjectWithReport(bucket *oss.Bucket, object string) error {
- err := rc.ossRestoreObject(bucket, object, "")
+ err := rc.ossRestoreObject(bucket, object, "", true)
rc.command.updateMonitor(err, &rc.monitor)
msg := fmt.Sprintf("restore %s", CloudURLToString(bucket.BucketName, object))
rc.command.report(msg, err, &rc.reOption)
@@ -414,3 +569,80 @@ func (rc *RestoreCommand) formatResultPrompt(err error) error {
}
return err
}
+
+func (rc *RestoreCommand) parseRestoreConf(xmlFile string) error {
+ fileInfo, err := os.Stat(xmlFile)
+ if err != nil {
+ return err
+ }
+
+ if fileInfo.IsDir() {
+ return fmt.Errorf("%s is dir,not the expected file", xmlFile)
+ }
+
+ if fileInfo.Size() == 0 {
+ return fmt.Errorf("%s is empty file", xmlFile)
+ }
+
+ // parsing the xml file
+ file, err := os.Open(xmlFile)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ text, err := ioutil.ReadAll(file)
+ if err != nil {
+ return err
+ }
+
+ rc.hasConfig = true
+ rc.configXml = string(text)
+ return nil
+}
+
+func (rc *RestoreCommand) formatSnapshotKey(bucket, object, msg string) string {
+ return CloudURLToString(bucket, object) + SnapshotConnector + msg
+}
+
+func (rc *RestoreCommand) skipRestore(spath string) bool {
+ if rc.reOption.snapshotPath != "" {
+ _, err := rc.reOption.snapshotldb.Get([]byte(spath), nil)
+ if err == nil {
+ return true
+ }
+ }
+ return false
+}
+
+func (rc *RestoreCommand) updateSnapshot(err error, spath string, srct int64) error {
+ if rc.reOption.snapshotPath != "" && err == nil {
+ srctstr := fmt.Sprintf("%d", srct)
+ err := rc.reOption.snapshotldb.Put([]byte(spath), []byte(srctstr), nil)
+ if err != nil {
+ return fmt.Errorf("dump snapshot error: %s", err.Error())
+ }
+ }
+ return nil
+}
+
+func (rc *RestoreCommand) checkObjectFile(objFileXml string) error {
+ // check file if exists
+ fileInfo, err := os.Stat(objFileXml)
+ if err != nil {
+ return err
+ }
+ if fileInfo.IsDir() {
+ return fmt.Errorf("%s is dir, not the expected file", objFileXml)
+ }
+ if fileInfo.Size() == 0 {
+ return fmt.Errorf("%s is empty file", objFileXml)
+ }
+
+ rc.hasObjFile = true
+ rc.objFilePath = objFileXml
+ return nil
+}
+
+func (rc *RestoreCommand) updateSkip(num int64) {
+ atomic.AddInt64(&rc.monitor.skipNum, num)
+}
diff --git a/lib/restore_test.go b/lib/restore_test.go
index 378a34c6..3f9d644d 100644
--- a/lib/restore_test.go
+++ b/lib/restore_test.go
@@ -3,19 +3,18 @@ package lib
import (
"encoding/xml"
"fmt"
+ oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
"net/url"
"os"
+ "strconv"
"strings"
-
- oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
- . "gopkg.in/check.v1"
)
func (s *OssutilCommandSuite) TestRestoreObject(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucketWithStorageClass(bucketName, StorageArchive, c)
- // put object to archive bucket
+ //put object to archive bucket
object := "恢复文件" + randStr(5)
data := randStr(20)
s.createFile(uploadFileName, data, c)
@@ -64,6 +63,647 @@ func (s *OssutilCommandSuite) TestRestoreObject(c *C) {
s.removeBucket(bucketName, true, c)
}
+func (s *OssutilCommandSuite) TestRestoreObjectErrorObj(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageColdArchive, c)
+ bucketNameIA := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketNameIA, StorageIA, c)
+
+ //put object to cold archive bucket
+ object := "恢复文件" + randStr(5)
+ s.putObject(bucketName, object, uploadFileName, c)
+ s.putObject(bucketNameIA, object, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketNameIA, object, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageIA)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ // not exist bucket
+ err := s.initRestoreObject([]string{CloudURLToString("xx", object)}, "", DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // not exist object
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, object+"xx")}, "", DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "StatusCode=404"), Equals, true)
+
+ // bucket is none
+ err = s.initRestoreObject([]string{CloudURLToString("", "")}, "", DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // error days
+ restoreXml := `
+
+ 99
+
+ xxx
+
+ `
+ restoreConfName := "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, object), restoreConfName}, "", DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // restore object(class ia)
+ err = s.initRestoreObject([]string{CloudURLToString(bucketNameIA, object)}, "", DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ s.removeBucket(bucketName, true, c)
+ s.removeBucket(bucketNameIA, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileBasic(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+
+ data := randStr(20)
+ s.createFile(uploadFileName, data, c)
+ object1 := "restore" + randStr(5)
+ object2 := "restore" + randStr(5)
+ object3 := "restore" + randStr(5)
+ object4 := "restore" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketName, object3, uploadFileName, c)
+ s.putObject(bucketName, object4, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ content := object1 + "\n" + object2 + "\n" + object3 + "\n" + object4
+ s.createFile(objectFileName, content, c)
+
+ err := s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileErrorObjFile(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+ bucketNameIA := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketNameIA, StorageIA, c)
+
+ data := randStr(20)
+ s.createFile(uploadFileName, data, c)
+ object1 := "restore" + randStr(5)
+ object2 := "restore" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketNameIA, object1, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketNameIA, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageIA)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ // file size 0
+ s.createFile(objectFileName, "", c)
+
+ err := s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+
+ // receive file but give dir
+ err = os.Mkdir(fmt.Sprintf("./%s", objectFileName), os.ModePerm)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+
+ // not exist file
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, fmt.Sprintf("--object-file %s -f", objectFileName+"xxx"), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // without -f
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, fmt.Sprintf("--object-file %s", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketNameIA, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageIA)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileErrVer(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+
+ object1 := "restore" + randStr(5)
+ object2 := "restore" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ command := "restore"
+ args := []string{CloudURLToString(bucketName, "")}
+ versionId := "xxx"
+ str := ""
+ routines := strconv.Itoa(Routines)
+ options := OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "versionId": &versionId,
+ "routines": &routines,
+ "objectFile": &objectFileName,
+ }
+ _, err := cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileWithConfCA(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageColdArchive, c)
+
+ // put object to coldArchive bucket
+ object1 := "restore" + randStr(5)
+ object2 := "restore" + randStr(5)
+ object3 := "restore" + randStr(5)
+ object4 := "restore" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketName, object3, uploadFileName, c)
+ s.putObject(bucketName, object4, uploadFileName, c)
+
+ // get object status
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ restoreXml := `
+
+ 2
+
+ Bulk
+
+ `
+
+ rulesConfigSrc := oss.RestoreConfiguration{}
+ err := xml.Unmarshal([]byte(restoreXml), &rulesConfigSrc)
+ c.Assert(err, IsNil)
+
+ restoreConfName := "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ // get object status
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ os.Remove(objectFileName)
+ os.Remove(restoreConfName)
+
+ // conf only with days
+ restoreXml = `
+
+ 7
+ `
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content = object3
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageColdArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ os.Remove(objectFileName)
+ os.Remove(restoreConfName)
+
+ // error Tier
+ restoreXml = `
+
+ 99
+
+ xxx
+
+ `
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content = object4
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f --disable-ignore-error", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // error Days
+ restoreXml = `
+
+ 399
+
+ Bulk
+
+ `
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content = object4
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f --disable-ignore-error", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // error xml
+ restoreXml = `
+
+ 4`
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f --disable-ignore-error", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ os.Remove(restoreConfName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileWithConfAr(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+
+ object1 := "restore" + randStr(5)
+ object2 := "restore" + randStr(5)
+ object3 := "restore" + randStr(5)
+ object4 := "restore" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketName, object3, uploadFileName, c)
+ s.putObject(bucketName, object4, uploadFileName, c)
+
+ // get object status
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ restoreXml := `
+
+ 7
+ `
+
+ rulesConfigSrc := oss.RestoreConfiguration{}
+ err := xml.Unmarshal([]byte(restoreXml), &rulesConfigSrc)
+ c.Assert(err, IsNil)
+
+ restoreConfName := "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ // get object status
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ os.Remove(restoreConfName)
+
+ // err conf
+ restoreXml = `
+
+ 2
+
+ Bulk
+
+ `
+
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content = object3
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f --disable-ignore-error", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "StatusCode=400"), Equals, true)
+
+ os.Remove(restoreConfName)
+
+ // err days
+ restoreXml = `
+
+ 99
+ `
+
+ rulesConfigSrc = oss.RestoreConfiguration{}
+ err = xml.Unmarshal([]byte(restoreXml), &rulesConfigSrc)
+ c.Assert(err, IsNil)
+
+ restoreConfName = "test-ossutil-" + randLowStr(12)
+ s.createFile(restoreConfName, restoreXml, c)
+
+ content = object4
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, ""), restoreConfName}, fmt.Sprintf("--object-file %s -f --disable-ignore-error", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "StatusCode=400"), Equals, true)
+
+ os.Remove(objectFileName)
+ os.Remove(restoreConfName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileSnap(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+
+ object1 := "restore_1"
+ object2 := "restore_2"
+ object3 := "restore_3"
+ object4 := "restore_4"
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketName, object3, uploadFileName, c)
+ s.putObject(bucketName, object4, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok := objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ snapShotDir := "ossutil_test_snapshot" + randStr(5)
+ cmd := fmt.Sprintf("--object-file %s -f --snapshot-path %s", objectFileName, snapShotDir)
+
+ err := s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ // get object status
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ _, ok = objectStat["X-Oss-Restore"]
+ c.Assert(ok, Equals, false)
+
+ os.Remove(objectFileName)
+
+ content = object1 + "\n" + object2 + "\n" + object3 + "\n" + object4
+ s.createFile(objectFileName, content, c)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ // get object status
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat["X-Oss-Storage-Class"], Equals, StorageArchive)
+ c.Assert(objectStat["X-Oss-Restore"], Equals, "ongoing-request=\"true\"")
+
+ os.Remove(objectFileName)
+ os.Remove(snapShotDir)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectFileErrorSnap(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+ bucketNameIA := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketNameIA, StorageIA, c)
+
+ object1 := "restore_1"
+ object2 := "restore_2"
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketNameIA, object1, uploadFileName, c)
+
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ // create file which name same as snapshotPath
+ snapShotDir := "ossutil_test_snapshot" + randStr(5)
+ s.createFile(snapShotDir, content, c)
+ cmd := fmt.Sprintf("--object-file %s -f --snapshot-path %s", objectFileName, snapShotDir)
+
+ err := s.initRestoreObject([]string{CloudURLToString(bucketName, "")}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ os.Remove(snapShotDir)
+
+ // restore object(class ia)
+ content = object1
+ s.createFile(objectFileName, content, c)
+
+ snapShotDir = "ossutil_test_snapshot" + randStr(5)
+
+ err = s.initRestoreObject([]string{CloudURLToString(bucketNameIA, "")}, fmt.Sprintf("--object-file %s -f --snapshot-path %s --disable-ignore-error", objectFileName, snapShotDir), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = restoreCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ os.Remove(snapShotDir)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestRestoreObjectWithVersionError(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+
+ //put object to archive bucket
+ object := "恢复文件" + randStr(5)
+ s.putObject(bucketName, object, uploadFileName, c)
+
+ // -r & --version-id error
+ command := "restore"
+ args := []string{CloudURLToString(bucketName, object)}
+ str := ""
+ versionId := "xxx"
+ r := true
+ options := OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "recursive": &r,
+ "versionId": &versionId,
+ }
+ _, err := cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ // only --version-id
+ options = OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "versionId": &versionId,
+ }
+ _, err = cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ s.removeBucket(bucketName, true, c)
+}
+
func (s *OssutilCommandSuite) TestRestoreObjectErrArgs(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
object := randStr(20)
@@ -419,6 +1059,53 @@ func (s *OssutilCommandSuite) TestRestoreObjectWithPayerError400(c *C) {
c.Assert(strings.Contains(err.Error(), "StatusCode=400"), Equals, true)
}
+func (s *OssutilCommandSuite) TestRestoreObjectWithPayer(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucketWithStorageClass(bucketName, StorageArchive, c)
+ s.createFile(uploadFileName, content, c)
+
+ //put object, with --payer=requester
+ args := []string{uploadFileName, CloudURLToString(bucketName, "")}
+ showElapse, err := s.rawCPWithPayer(args, false, true, false, DefaultBigFileThreshold, "requester")
+ c.Assert(err, IsNil)
+ c.Assert(showElapse, Equals, true)
+
+ // stat with payer
+ command := "restore"
+ args = []string{CloudURLToString(bucketName, uploadFileName)}
+ str := ""
+ requester := "requester"
+ options := OptionMapType{
+ "endpoint": &payerBucketEndPoint,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "payer": &requester,
+ }
+ _, err = cm.RunCommand(command, args, options)
+ c.Assert(err, IsNil)
+
+ //put object, with --payer=requester
+ args = []string{uploadFileName, CloudURLToString(bucketName, "")}
+ showElapse, err = s.rawCPWithPayer(args, false, true, false, DefaultBigFileThreshold, "requester")
+ c.Assert(err, IsNil)
+ c.Assert(showElapse, Equals, true)
+
+ // stat with payer
+ requester = "request"
+ options = OptionMapType{
+ "endpoint": &payerBucketEndPoint,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "configFile": &configFile,
+ "payer": &requester,
+ }
+ _, err = cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+}
+
func (s *OssutilCommandSuite) TestRestoreObjectWithConfigColdArchiveSuccess(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucketWithStorageClass(bucketName, StorageColdArchive, c)
diff --git a/lib/set_meta.go b/lib/set_meta.go
index 38e3b8be..6ef3137c 100644
--- a/lib/set_meta.go
+++ b/lib/set_meta.go
@@ -1,7 +1,9 @@
package lib
import (
+ "bufio"
"fmt"
+ "github.com/syndtr/goleveldb/leveldb"
"net/http"
"os"
"strings"
@@ -85,7 +87,7 @@ var specChineseSetMeta = SpecText{
paramText: "cloud_url [meta] [options]",
syntaxText: `
- ossutil set-meta oss://bucket[/prefix] [header:value#header:value...] [--update] [--delete] [-r] [-f] [-c file] [--version-id versionId]
+ ossutil set-meta oss://bucket[/prefix] [header:value#header:value...] [--update] [--delete] [-r] [-f] [-c file] [--version-id versionId] [--object-file file] [--snapshot-path dir] [--disable-ignore-error]
`,
detailHelpText: `
@@ -137,6 +139,16 @@ Headers:
--include和--exclude选项说明，请参考cp命令帮助。
如果--force选项被指定，则不会进行询问提示。
--update选项和--delete选项的用法参考上文。
+
+ 3) ossutil set-meta oss://bucket [header:value#header:value...] --object-file file [--snapshot-path dir] [--disable-ignore-error] [--update] [--delete] [-f]
+ 如果指定了--object-file选项，ossutil会读取指定文件中的所有objects，批量设置
+ 这些objects的meta信息。当一个object操作出现错误时会将出错object的错误信息记录到report
+ 文件，并继续操作其他object，成功操作的object信息将不会被记录到report文件中（更多信息
+ 见cp命令的帮助）。
+ 如果--snapshot-path选项被指定，则会对本次操作的object进行快照，如果操作对象已经存在
+ 快照，则忽略本次操作。（仅支持在-r、--object-file基础上）
+ 如果--force选项被指定，则不会进行询问提示。
+ --update选项和--delete选项的用法参考上文。
`,
sampleText: `
@@ -161,8 +173,14 @@ Headers:
(7)ossutil set-meta oss://bucket1/%e4%b8%ad%e6%96%87 X-Oss-Meta-delete --delete --encoding-type url
删除oss://bucket1/中文的X-Oss-Meta-delete头域
- (6)ossutil set-meta oss://bucket1/obj1 X-Oss-Meta-delete --delete --version-id versionId
+ (8)ossutil set-meta oss://bucket1/obj1 X-Oss-Meta-delete --delete --version-id versionId
删除指定版本obj1的X-Oss-Meta-delete头域，并生成最新版本
+
+ (9)ossutil set-meta oss://bucket1 X-Oss-Meta-empty:#Content-Type:plain/text --update --object-file file
+ 批量更新file文件中所有objects的X-Oss-Meta-empty和Content-Type头域
+
+ (10)ossutil set-meta oss://bucket1 X-Oss-Meta-empty:#Content-Type:plain/text --update --object-file file --snapshot-path dir
+ 批量更新file文件中所有objects的X-Oss-Meta-empty和Content-Type头域，并开启快照
`,
}
@@ -173,7 +191,7 @@ var specEnglishSetMeta = SpecText{
paramText: "cloud_url [meta] [options]",
syntaxText: `
- ossutil set-meta oss://bucket[/prefix] [header:value#header:value...] [--update] [--delete] [-r] [-f] [-c file] [--version-id versionId]
+ ossutil set-meta oss://bucket[/prefix] [header:value#header:value...] [--update] [--delete] [-r] [-f] [-c file] [--version-id versionId] [--object-file file] [--snapshot-path dir] [--disable-ignore-error]
`,
detailHelpText: `
@@ -233,6 +251,25 @@ Usage:
--include and --exclude option, please refer cp command help.
If --force option is specified, ossutil will not show prompt question.
The usage of --update option and --delete option is showed in detailHelpText.
+
+ 3) ossutil set-meta oss://bucket [header:value#header:value...] --object-file file [--snapshot-path dir] [--disable-ignore-error] [--update] [--delete] [-f]
+ 如果指定了--object-file选项，ossutil会读取指定文件中的所有objects，批量设置
+ 这些objects的meta信息。当一个object操作出现错误时会将出错object的错误信息记录到report
+ 文件，并继续操作其他object，成功操作的object信息将不会被记录到report文件中（更多信息
+ 见cp命令的帮助）。
+ 如果--snapshot-path选项被指定，则会对本次操作的object进行快照，如果操作对象已经存在
+ 快照，则忽略本次操作。（仅支持在-r、--object-file基础上）
+ 如果--force选项被指定，则不会进行询问提示。
+ --update选项和--delete选项的用法参考上文。
+
+ If --object-file option is specified, ossutil will read objects in file, then
+ set meta on these objects. If an error occurs, ossutil will record the error message
+ to report file, and ossutil will continue to attempt to set acl on the remaining objects(
+ more information see help of cp command).
+ If --snapshot-path option is specified, ossutil will create snapshot for this operation,
+ and if the snapshot exists, then cancel this operate.
+ If --force option is specified, ossutil will not show prompt question.
+ The usage of --update option and --delete option is showed in detailHelpText.
`,
sampleText: `
@@ -259,16 +296,24 @@ Usage:
(8)ossutil set-meta oss://bucket1/obj1 X-Oss-Meta-delete --delete --version-id versionId
Delete X-Oss-Meta-delete header of a specific version of obj1，and generate the latest version obj1
+
+ (9)ossutil set-meta oss://bucket1 X-Oss-Meta-empty:#Content-Type:plain/text --update --object-file file
+ Batch update X-Oss-Meta-empty and Content-Type header on objects that in file
+
+ (10)ossutil set-meta oss://bucket1 X-Oss-Meta-empty:#Content-Type:plain/text --update --object-file file --snapshot-path dir
+ Batch update X-Oss-Meta-empty and Content-Type header on objects that in file, and open snapshot
`,
}
// SetMetaCommand is the command set meta for object
type SetMetaCommand struct {
- monitor Monitor //Put first for atomic op on some fileds
- command Command
- smOption batchOptionType
- filters []filterOptionType
- skipCount uint64
+ monitor Monitor //Put first for atomic op on some fileds
+ command Command
+ smOption batchOptionType
+ filters []filterOptionType
+ skipCount uint64
+ hasObjFile bool
+ objFilePath string
}
var setMetaCommand = SetMetaCommand{
@@ -313,6 +358,9 @@ var setMetaCommand = SetMetaCommand{
OptionSTSRegion,
OptionSkipVerfiyCert,
OptionUserAgent,
+ OptionObjectFile,
+ OptionSnapshotPath,
+ OptionDisableIgnoreError,
},
},
}
@@ -334,7 +382,6 @@ func (sc *SetMetaCommand) Init(args []string, options OptionMapType) error {
// RunCommand simulate inheritance, and polymorphism
func (sc *SetMetaCommand) RunCommand() error {
sc.monitor.init("Setted meta on")
-
isUpdate, _ := GetBool(OptionUpdate, sc.command.options)
isDelete, _ := GetBool(OptionDelete, sc.command.options)
recursive, _ := GetBool(OptionRecursion, sc.command.options)
@@ -344,35 +391,31 @@ func (sc *SetMetaCommand) RunCommand() error {
language = strings.ToLower(language)
encodingType, _ := GetString(OptionEncodingType, sc.command.options)
versionId, _ := GetString(OptionVersionId, sc.command.options)
+ objFileXml, _ := GetString(OptionObjectFile, sc.command.options)
+ snapshotPath, _ := GetString(OptionSnapshotPath, sc.command.options)
- var res bool
- res, sc.filters = getFilter(os.Args)
- if !res {
- return fmt.Errorf("--include or --exclude does not support format containing dir info")
- }
-
- if !recursive && len(sc.filters) > 0 {
- return fmt.Errorf("--include or --exclude only work with --recursive")
- }
-
- if recursive && len(versionId) > 0 {
- return fmt.Errorf("--version-id only work on single object")
+ var err error
+ // load snapshot
+ sc.smOption.snapshotPath = snapshotPath
+ if sc.smOption.snapshotPath != "" {
+ if sc.smOption.snapshotldb, err = leveldb.OpenFile(sc.smOption.snapshotPath, nil); err != nil {
+ return fmt.Errorf("load snapshot error, reason: %s", err.Error())
+ }
+ defer sc.smOption.snapshotldb.Close()
}
cloudURL, err := CloudURLFromString(sc.command.args[0], encodingType)
if err != nil {
return err
}
-
- if err = sc.checkArgs(cloudURL, recursive, isUpdate, isDelete); err != nil {
+ if err := sc.checkOptions(cloudURL, isUpdate, isDelete, force, recursive, language, versionId, objFileXml); err != nil {
+ if err.Error() == "operation is canceled by user" {
+ return nil
+ }
return err
}
-
- if !sc.confirmOP(recursive, force) {
- return nil
- }
-
- if err := sc.checkOption(isUpdate, isDelete, force, language); err != nil {
+ bucket, err := sc.command.ossBucket(cloudURL.bucket)
+ if err != nil {
return err
}
@@ -380,21 +423,27 @@ func (sc *SetMetaCommand) RunCommand() error {
if err != nil {
return err
}
-
headers, err := sc.command.parseHeaders(str, isDelete)
if err != nil {
return err
}
- bucket, err := sc.command.ossBucket(cloudURL.bucket)
- if err != nil {
- return err
- }
+ sc.smOption.ctnu = true
- if !recursive {
- err = sc.setObjectMeta(bucket, cloudURL.object, headers, isUpdate, isDelete, versionId)
+ // check --object-file mode
+ if objFileXml != "" {
+ // check objFileXml and parse it
+ if err := sc.checkObjectFile(objFileXml); err != nil {
+ return err
+ }
+ recursive = true
+ err = sc.batchSetObjectsMetaFromFile(bucket, cloudURL, headers, isUpdate, isDelete, recursive, routines)
} else {
- err = sc.batchSetObjectMeta(bucket, cloudURL, headers, isUpdate, isDelete, force, routines)
+ if !recursive {
+ err = sc.setObjectMeta(bucket, cloudURL.object, headers, isUpdate, isDelete, false, versionId)
+ } else {
+ err = sc.batchSetObjectMeta(bucket, cloudURL, headers, isUpdate, isDelete, force, routines)
+ }
}
if isUpdate {
@@ -403,20 +452,47 @@ func (sc *SetMetaCommand) RunCommand() error {
return err
}
-func (sc *SetMetaCommand) checkArgs(cloudURL CloudURL, recursive, isUpdate, isDelete bool) error {
+func (sc *SetMetaCommand) checkOptions(cloudURL CloudURL, isUpdate, isDelete, force, recursive bool, language, versionId, objFileXml string) error {
if cloudURL.bucket == "" {
- return fmt.Errorf("invalid cloud url: %s, miss bucket", sc.command.args[0])
+ return fmt.Errorf("invalid cloud url: %s, miss bucket", cloudURL.urlStr)
+ }
+ if cloudURL.object == "" {
+ if !recursive && objFileXml == "" {
+ return fmt.Errorf("set object meta invalid cloud url: %s, object empty. Set bucket meta is not supported, if you mean batch set meta on objects, please use --recursive or --object-file", sc.command.args[0])
+ }
+ } else {
+ if objFileXml != "" {
+ return fmt.Errorf("the first arg of `ossutil set-meta` only support oss://bucket when set option --object-file")
+ }
+ }
+
+ var res bool
+ res, sc.filters = getFilter(os.Args)
+ if !res {
+ return fmt.Errorf("--include or --exclude does not support format containing dir info")
}
- if !recursive && cloudURL.object == "" {
- return fmt.Errorf("set object meta invalid cloud url: %s, object empty. Set bucket meta is not supported, if you mean batch set meta on objects, please use --recursive", sc.command.args[0])
+
+ if !recursive && len(sc.filters) > 0 {
+ return fmt.Errorf("--include or --exclude only work with --recursive")
}
+
+ if (recursive && len(versionId) > 0) || (objFileXml != "" && len(versionId) > 0) {
+ return fmt.Errorf("--version-id only work on single object")
+ }
+
+ if !force {
+ var val string
+ if recursive || objFileXml != "" {
+ fmt.Printf("Do you really mean to recursivlly set meta on objects of %s(y or N)? ", sc.command.args[0])
+ if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
+ return fmt.Errorf("operation is canceled by user")
+ }
+ }
+ }
+
if isUpdate && isDelete {
return fmt.Errorf("--update option and --delete option are not supported for %s at the same time, please check", sc.command.args[0])
}
- return nil
-}
-
-func (sc *SetMetaCommand) checkOption(isUpdate, isDelete, force bool, language string) error {
if !isUpdate && !isDelete && !force {
if language == LEnglishLanguage {
fmt.Printf("Warning: --update option means update the specified header, --delete option means delete the specified header, miss both options means update the whole meta info, continue to update the whole meta info(y or N)? ")
@@ -432,18 +508,6 @@ func (sc *SetMetaCommand) checkOption(isUpdate, isDelete, force bool, language s
return nil
}
-func (sc *SetMetaCommand) confirmOP(recursive, force bool) bool {
- if recursive && !force {
- var val string
- fmt.Printf("Do you really mean to recursivlly set meta on objects of %s(y or N)? ", sc.command.args[0])
- if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
- fmt.Println("operation is canceled.")
- return false
- }
- }
- return true
-}
-
func (sc *SetMetaCommand) getMetaData(force bool, language string) (string, error) {
if len(sc.command.args) > 1 {
return strings.TrimSpace(sc.command.args[1]), nil
@@ -494,7 +558,7 @@ func (cmd *Command) parseHeaders(str string, isDelete bool) (map[string]string,
if isDelete && value != "" {
return nil, fmt.Errorf("delete meta for object do no support value for header:%s, please set value:%s to empty", name, value)
}
- if _, err := fetchHeaderOptionMap(headerOptionMap, name); err != nil && !strings.HasPrefix(strings.ToLower(name), "x-oss-") {
+ if _, err := fetchHeaderOptionMap(headerOptionMap, name); err != nil && !strings.HasPrefix(strings.ToLower(name), strings.ToLower(oss.HTTPHeaderOssMetaPrefix)) {
return nil, fmt.Errorf("unsupported header:%s, please try \"help %s\" to see supported headers", name, cmd.name)
}
headers[name] = value
@@ -502,9 +566,22 @@ func (cmd *Command) parseHeaders(str string, isDelete bool) (map[string]string,
return headers, nil
}
-func (sc *SetMetaCommand) setObjectMeta(bucket *oss.Bucket, object string, headers map[string]string, isUpdate, isDelete bool, versionId string) error {
+func (sc *SetMetaCommand) setObjectMeta(bucket *oss.Bucket, object string, headers map[string]string, isUpdate, isDelete, batchOperate bool, versionId string) error {
allheaders := headers
isSkip := false
+ spath := ""
+ msg := "set_meta"
+ nowt := time.Now().Unix()
+
+ if batchOperate && sc.smOption.snapshotPath != "" {
+ spath = sc.formatSnapshotKey(bucket.BucketName, object, msg)
+ if skip := sc.skipSetMeta(spath); skip {
+ sc.updateSkip(1)
+ LogInfo("restore obj skip: %s\n", object)
+ return nil
+ }
+ }
+
if isUpdate || isDelete {
var options []oss.Option
if len(versionId) > 0 {
@@ -539,7 +616,23 @@ func (sc *SetMetaCommand) setObjectMeta(bucket *oss.Bucket, object string, heade
if len(versionId) > 0 {
options = append(options, oss.VersionId(versionId))
}
- return sc.ossSetObjectMetaRetry(bucket, object, options...)
+
+ err = sc.ossSetObjectMetaRetry(bucket, object, options...)
+ if batchOperate && sc.smOption.snapshotPath != "" {
+ if err != nil {
+ _ = sc.updateSnapshot(err, spath, nowt)
+ return err
+ } else {
+ err = sc.updateSnapshot(err, spath, nowt)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ return err
+ }
+
+ return nil
}
func (sc *SetMetaCommand) mergeHeader(props http.Header, headers map[string]string, isUpdate, isDelete bool) (map[string]string, bool) {
@@ -582,10 +675,11 @@ func (sc *SetMetaCommand) mergeHeader(props http.Header, headers map[string]stri
func (sc *SetMetaCommand) ossSetObjectMetaRetry(bucket *oss.Bucket, object string, options ...oss.Option) error {
retryTimes, _ := GetInt(OptionRetryTimes, sc.command.options)
cpOptions := append(options, oss.MetadataDirective(oss.MetaReplace))
+
for i := 1; ; i++ {
_, err := bucket.CopyObject(object, object, cpOptions...)
if err == nil {
- return err
+ return nil
}
if int64(i) >= retryTimes {
return ObjectError{err, bucket.BucketName, object}
@@ -594,7 +688,6 @@ func (sc *SetMetaCommand) ossSetObjectMetaRetry(bucket *oss.Bucket, object strin
}
func (sc *SetMetaCommand) batchSetObjectMeta(bucket *oss.Bucket, cloudURL CloudURL, headers map[string]string, isUpdate, isDelete, force bool, routines int64) error {
- sc.smOption.ctnu = true
outputDir, _ := GetString(OptionOutputDir, sc.command.options)
// init reporter
@@ -639,7 +732,7 @@ func (sc *SetMetaCommand) setObjectMetaConsumer(bucket *oss.Bucket, headers map[
}
func (sc *SetMetaCommand) setObjectMetaWithReport(bucket *oss.Bucket, object string, headers map[string]string, isUpdate, isDelete bool) error {
- err := sc.setObjectMeta(bucket, object, headers, isUpdate, isDelete, "")
+ err := sc.setObjectMeta(bucket, object, headers, isUpdate, isDelete, true, "")
sc.command.updateMonitor(err, &sc.monitor)
msg := fmt.Sprintf("set meta on %s", CloudURLToString(bucket.BucketName, object))
sc.command.report(msg, err, &sc.smOption)
@@ -678,3 +771,131 @@ func (sc *SetMetaCommand) formatResultPrompt(err error) error {
}
return err
}
+
+func (sc *SetMetaCommand) checkObjectFile(objFileXml string) error {
+ // check file if exists
+ fileInfo, err := os.Stat(objFileXml)
+ if err != nil {
+ return err
+ }
+ if fileInfo.IsDir() {
+ return fmt.Errorf("%s is dir, not the expected file", objFileXml)
+ }
+ if fileInfo.Size() == 0 {
+ return fmt.Errorf("%s is empty file", objFileXml)
+ }
+
+ sc.hasObjFile = true
+ sc.objFilePath = objFileXml
+ return nil
+}
+
+func (sc *SetMetaCommand) batchSetObjectsMetaFromFile(bucket *oss.Bucket, cloudURL CloudURL, headers map[string]string, isUpdate, isDelete, recursive bool, routines int64) error {
+ if sc.hasObjFile || recursive {
+ disableIgnoreError, _ := GetBool(OptionDisableIgnoreError, sc.command.options)
+ sc.smOption.ctnu = !disableIgnoreError
+ }
+ outputDir, _ := GetString(OptionOutputDir, sc.command.options)
+
+ // init reporter
+ var err error
+ if sc.smOption.reporter, err = GetReporter(sc.smOption.ctnu, outputDir, commandLine); err != nil {
+ return err
+ }
+ defer sc.smOption.reporter.Clear()
+
+ return sc.setObjectsMetaFromFile(bucket, cloudURL, sc.objFilePath, headers, isUpdate, isDelete, routines)
+}
+
+func (sc *SetMetaCommand) setObjectsMetaFromFile(bucket *oss.Bucket, cloudURL CloudURL, objectFile string, headers map[string]string, isUpdate, isDelete bool, routines int64) error {
+ // producer list objects
+ // consumer set meta
+ chObjects := make(chan string, ChannelBuf)
+ chError := make(chan error, routines+1)
+ chListError := make(chan error, 1)
+ go sc.setObjectMetaStatistic(objectFile, &sc.monitor, sc.filters)
+ go sc.setObjectMetaProducer(objectFile, chObjects, chListError, sc.filters)
+ for i := 0; int64(i) < routines; i++ {
+ go sc.setObjectMetaConsumer(bucket, headers, isUpdate, isDelete, chObjects, chError)
+ }
+
+ return sc.waitRoutinueComplete(chError, chListError, routines)
+}
+
+func (sc *SetMetaCommand) setObjectMetaStatistic(objectFile string, monitor Monitorer, filters []filterOptionType, options ...oss.Option) {
+ if monitor == nil {
+ return
+ }
+
+ file, err := os.Open(objectFile)
+ if err != nil {
+ monitor.setScanError(err)
+ return
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ object := scanner.Text()
+ object = strings.Trim(object, " ")
+ if object == "" {
+ monitor.setScanError(fmt.Errorf("object can't be '' in --object-file"))
+ return
+ }
+ monitor.updateScanNum(1)
+ }
+
+ monitor.setScanEnd()
+}
+
+func (sc *SetMetaCommand) setObjectMetaProducer(objectFile string, chObjects chan<- string, chError chan<- error, filters []filterOptionType, options ...oss.Option) {
+ file, err := os.Open(objectFile)
+ if err != nil {
+ chError <- err
+ return
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ object := scanner.Text()
+ object = strings.Trim(object, " ")
+ if object == "" {
+ chError <- fmt.Errorf("object can't be '' in --object-file")
+ break
+ }
+ chObjects <- object
+ }
+
+ defer close(chObjects)
+ chError <- nil
+}
+
+func (sc *SetMetaCommand) formatSnapshotKey(bucket, object, msg string) string {
+ return CloudURLToString(bucket, object) + SnapshotConnector + msg
+}
+
+func (sc *SetMetaCommand) skipSetMeta(spath string) bool {
+ if sc.smOption.snapshotPath != "" {
+ _, err := sc.smOption.snapshotldb.Get([]byte(spath), nil)
+ if err == nil {
+ return true
+ }
+ }
+ return false
+}
+
+func (sc *SetMetaCommand) updateSnapshot(err error, spath string, srct int64) error {
+ if sc.smOption.snapshotPath != "" && err == nil {
+ srctstr := fmt.Sprintf("%d", srct)
+ err := sc.smOption.snapshotldb.Put([]byte(spath), []byte(srctstr), nil)
+ if err != nil {
+ return fmt.Errorf("dump snapshot error: %s", err.Error())
+ }
+ }
+ return nil
+}
+
+func (sc *SetMetaCommand) updateSkip(num int64) {
+ atomic.AddInt64(&sc.monitor.skipNum, num)
+}
diff --git a/lib/set_meta_test.go b/lib/set_meta_test.go
index d4a90174..8861b138 100644
--- a/lib/set_meta_test.go
+++ b/lib/set_meta_test.go
@@ -2,16 +2,15 @@ package lib
import (
"fmt"
+ oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ . "gopkg.in/check.v1"
"net/url"
"os"
"strconv"
"strings"
-
- oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
- . "gopkg.in/check.v1"
)
-func (s *OssutilCommandSuite) TestSetBucketMeta(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaSetBucketMeta(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -79,6 +78,11 @@ func (s *OssutilCommandSuite) TestSetObjectMetaBasic(c *C) {
c.Assert(err, NotNil)
c.Assert(showElapse, Equals, false)
+ // with update and delete
+ showElapse, err = s.rawSetMeta(bucketName, object, "x-oss-object-acl:default#X-Oss-Meta-A:A", true, true, false, true, DefaultLanguage)
+ c.Assert(err, NotNil)
+ c.Assert(showElapse, Equals, false)
+
// miss meta
s.setObjectMeta(bucketName, object, "", true, false, false, true, c)
@@ -135,7 +139,358 @@ func (s *OssutilCommandSuite) TestSetObjectMetaBasic(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestSetNotExistObjectMeta(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFile(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok := objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ // --object-file without -f
+ err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFileErrMeta(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ meta := "x-oss-server-side-encryption:xxx"
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "StatusCode=400"), Equals, true)
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFileErrObj(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+ content := object1 + "xx" + "\n" + object2 + "ss"
+ s.createFile(objectFileName, content, c)
+
+ // not exist object
+ err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // bucket is none
+ err = s.initSetMetaWithArgs([]string{CloudURLToString("", ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // object is none
+ content = "\n"
+ s.createFile(objectFileName, content, c)
+
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // file size 0
+ s.createFile(objectFileName, "", c)
+
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+
+ // receive file but get dir
+ err = os.Mkdir(fmt.Sprintf("./%s", objectFileName), os.ModePerm)
+
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+
+ // not exist file
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ // object exist while --object-file is set
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, "xx"), meta}, fmt.Sprintf("--object-file %s --disable-ignore-error -f", objectFileName), DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFileErrVer(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ command := "set-meta"
+ args := []string{CloudURLToString(bucketName, ""), meta}
+ versionId := "xxx"
+ str := ""
+ ok := true
+ routines := strconv.Itoa(Routines)
+ options := OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "update": &ok,
+ "force": &ok,
+ "versionId": &versionId,
+ "routines": &routines,
+ "objectFile": &objectFileName,
+ "disableIgnoreError": &ok,
+ }
+ _, err := cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFileSnap(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ object3 := "setMeta" + randStr(5)
+ object4 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+ s.putObject(bucketName, object3, uploadFileName, c)
+ s.putObject(bucketName, object4, uploadFileName, c)
+
+ objectStat := s.getStat(bucketName, object1, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok := objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ snapShotDir := "ossutil_test_snapshot" + randStr(5)
+ cmd := fmt.Sprintf("--object-file %s --snapshot-path %s -f", objectFileName, snapShotDir)
+
+ err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object1, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ objectStat = s.getStat(bucketName, object2, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat[StatACL], Equals, "default")
+ _, ok = objectStat["X-Oss-Meta-A"]
+ c.Assert(ok, Equals, false)
+
+ content = object1 + "\n" + object2 + "\n" + object3 + "\n" + object4
+ s.createFile(objectFileName, content, c)
+
+ err = s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, IsNil)
+
+ objectStat = s.getStat(bucketName, object3, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ objectStat = s.getStat(bucketName, object4, c)
+ c.Assert(objectStat[StatACL], Equals, "private")
+ c.Assert(objectStat["X-Oss-Meta-A"], Equals, "A")
+ c.Assert(objectStat["Expires"], Equals, "Mon, 02 Jan 2006 15:04:05 GMT")
+
+ os.Remove(objectFileName)
+ os.Remove(snapShotDir)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaObjectFileErrorSnap(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta_" + randStr(5)
+ object2 := "setMeta_" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+ content := object1 + "\n" + object2
+ s.createFile(objectFileName, content, c)
+
+ // create file which name same as snapshotPath
+ snapShotDir := "ossutil_test_snapshot" + randStr(5)
+ s.createFile(snapShotDir, content, c)
+ cmd := fmt.Sprintf("--object-file %s --snapshot-path %s --disable-ignore-error -f", objectFileName, snapShotDir)
+
+ err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketName, ""), meta}, cmd, DefaultOutputDir)
+ c.Assert(err, IsNil)
+ err = setMetaCommand.RunCommand()
+ c.Assert(err, NotNil)
+
+ os.Remove(objectFileName)
+ os.Remove(snapShotDir)
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaWithVersionError(c *C) {
+ bucketName := bucketNamePrefix + randLowStr(10)
+ s.putBucket(bucketName, c)
+
+ object1 := "setMeta" + randStr(5)
+ object2 := "setMeta" + randStr(5)
+ s.putObject(bucketName, object1, uploadFileName, c)
+ s.putObject(bucketName, object2, uploadFileName, c)
+
+ meta := "x-oss-object-acl:private#X-Oss-Meta-A:A#Expires:2006-01-02T15:04:05Z"
+
+ // -r & --version-id error
+ command := "set-meta"
+ args := []string{CloudURLToString(bucketName, ""), meta}
+ str := ""
+ versionId := "xxx"
+ r := true
+ options := OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "recursive": &r,
+ "update": &r,
+ "force": &r,
+ "versionId": &versionId,
+ }
+ _, err := cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ // wrong version-id
+ options = OptionMapType{
+ "endpoint": &str,
+ "accessKeyID": &str,
+ "accessKeySecret": &str,
+ "stsToken": &str,
+ "update": &r,
+ "force": &r,
+ "versionId": &versionId,
+ }
+ _, err = cm.RunCommand(command, args, options)
+ c.Assert(err, NotNil)
+
+ s.removeBucket(bucketName, true, c)
+}
+
+func (s *OssutilCommandSuite) TestSetObjectMetaSetNotExistObjectMeta(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -164,7 +519,7 @@ func (s *OssutilCommandSuite) TestSetNotExistObjectMeta(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestErrBatchSetMeta(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaErrBatchSetMeta(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -209,7 +564,7 @@ func (s *OssutilCommandSuite) TestErrBatchSetMeta(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestErrSetMeta(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaErrSetMeta(c *C) {
args := []string{"os:/", ""}
showElapse, err := s.rawSetMetaWithArgs(args, false, false, false, true, DefaultLanguage)
c.Assert(err, NotNil)
@@ -267,12 +622,12 @@ func (s *OssutilCommandSuite) TestErrSetMeta(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestGetOSSOption(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaGetOSSOption(c *C) {
_, err := getOSSOption(headerOptionMap, "unknown", "a")
c.Assert(err, NotNil)
}
-func (s *OssutilCommandSuite) TestSetMetaIDKey(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaSetMetaIDKey(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -320,7 +675,7 @@ func (s *OssutilCommandSuite) TestSetMetaIDKey(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestSetMetaURLEncoding(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaSetMetaURLEncoding(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -357,7 +712,7 @@ func (s *OssutilCommandSuite) TestSetMetaURLEncoding(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestSetMetaErrArgs(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaSetMetaErrArgs(c *C) {
object := randStr(20)
meta := "x-oss-object-acl:public-read-write"
@@ -372,7 +727,7 @@ func (s *OssutilCommandSuite) TestSetMetaErrArgs(c *C) {
c.Assert(err, NotNil)
}
-func (s *OssutilCommandSuite) TestBatchSetMetaNotExistBucket(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaBatchSetMetaNotExistBucket(c *C) {
// set acl notexist bucket
meta := "x-oss-object-acl:public-read-write"
err := s.initSetMetaWithArgs([]string{CloudURLToString(bucketNamePrefix+randLowStr(10), ""), meta}, "-rf", DefaultOutputDir)
@@ -381,7 +736,7 @@ func (s *OssutilCommandSuite) TestBatchSetMetaNotExistBucket(c *C) {
c.Assert(err, NotNil)
}
-func (s *OssutilCommandSuite) TestBatchSetMetaErrorContinue(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaBatchSetMetaErrorContinue(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucket(bucketName, c)
@@ -478,7 +833,7 @@ func (s *OssutilCommandSuite) TestBatchSetMetaErrorContinue(c *C) {
s.removeBucket(bucketName, true, c)
}
-func (s *OssutilCommandSuite) TestBatchSetMetaErrorBreak(c *C) {
+func (s *OssutilCommandSuite) TestSetObjectMetaBatchSetMetaErrorBreak(c *C) {
bucketName := bucketNamePrefix + randLowStr(10)
s.putBucketWithStorageClass(bucketName, StorageArchive, c)
@@ -610,6 +965,16 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithNormalInclude(c *C) {
c.Assert(ok, Equals, false)
}
+ // --include without -r
+ showElapse, err = s.rawSetMetaWithFilter(args, true, false, false, true, DefaultLanguage, []string{"ossutil", "set-meta", bucketStr, meta, "-f", "--include", "*.txt"})
+ c.Assert(err, NotNil)
+ c.Assert(showElapse, Equals, false)
+
+ // error value of --include
+ showElapse, err = s.rawSetMetaWithFilter(args, true, false, false, true, DefaultLanguage, []string{"ossutil", "set-meta", bucketStr, meta, "-f", "--include", "/usr/test/*.txt"})
+ c.Assert(err, NotNil)
+ c.Assert(showElapse, Equals, false)
+
os.RemoveAll(dir)
s.removeBucket(bucketName, true, c)
}
@@ -1343,8 +1708,6 @@ func (s *OssutilCommandSuite) TestSetObjectMetaWithInvalidIncExc(c *C) {
func (s *OssutilCommandSuite) TestSetObjectMetaVersionBasic(c *C) {
bucketName := bucketNamePrefix + "-set-meta-" + randLowStr(10)
- s.putBucket(bucketName, c)
-
s.putBucket(bucketName, c)
s.putBucketVersioning(bucketName, "enabled", c)