AliOss文件遷移到JDOss

因爲項目需要把AliOss上的許多文件遷移到JDOss上，本來JDoss上有個工具可以實現遷移的，但是實際效果卻不怎麼好，很多文件都遺漏了，所以針對這種情況寫了一個函數來判斷缺少了哪些函數，缺少的從Ali下載到JD上，實現思路：因爲是判斷兩個雲上文件的差異，所以先根據prefix遍歷Ali文件夾，將文件key作爲key，size作爲value緩存到內存中，在根據同樣的prefix遍歷JD雲上的文件，同樣緩存到map中，將Alimap的key取出一一和JD對應，JD沒有的文件就從Ali下載，得到的文件流直接上傳相同目錄到JDOss，下面是具體代碼的實現：

---

//將AliOss的文件拷貝至JDOss
func CopyAliOss2JDoss() {

	//列舉文件使用的prefix
	bothverdsion := "2012"

	a := ListObjectinstance{}
	a.aliListmap = map[string]int64{}
	a.JDListmap = map[string]int64{}
	a.shrotmap = map[string]int64{}

	// 創建OSSClient實例。
	client, err := oss.New("xxxxxxxxxxxxxxx", "xxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxx")
	if err != nil {
		fmt.Println("創建oss上傳client Error:", err)
		return
	}
	// 獲取存儲空間。
	bucket, err := client.Bucket("xxxxx")
	if err != nil {
		fmt.Println("創建oss上傳bucket Error:", err)
		//os.Exit(-1)
		return
	}
	//分頁列舉文件
	marker := oss.Marker("")
	prefix := oss.Prefix(bothverdsion)
	//fmt.Println("列舉文件:", prefix)
	for {
		lsRes, err := bucket.ListObjects(marker, prefix)
		if err != nil {
			fmt.Println("從oss獲取文件失敗", err)
			return
		}
		for _, object := range lsRes.Objects {
			//下載文件到working目錄
			//fmt.Println("key:", object.Key)
			a.aliListmap[object.Key] = object.Size
		}
		prefix = oss.Prefix(lsRes.Prefix)
		marker = oss.Marker(lsRes.NextMarker)
		if !lsRes.IsTruncated {
			break
		}
	}
	logs.Info(" 獲得alimap的size:", len(a.aliListmap))

	JDbucket := "xxxxxx"
	JDPrefix := bothverdsion
	JDMarker := ""
	input := s3.ListObjectsInput{
		Bucket: &JDbucket,
		Prefix: &JDPrefix,
		Marker: &JDMarker,
	}
	Client := NewS3Client()
	for {
		output, err := Client.ListObjects(&input)
		if err != nil {
			logs.Error("獲取JD失敗:", err.Error())
			return
		}

		for _, value := range output.Contents {
			crtDir := *value.Key
			size := *value.Size
			a.JDListmap[crtDir] = size
		}
		input.Marker = output.NextMarker
		input.Prefix = output.Prefix
		if !*output.IsTruncated {
			break
		}
	}
	logs.Info(" 獲得JDmap的size:", len(a.JDListmap))

	for k, v := range a.aliListmap {
		if jd, ok := a.JDListmap[k]; ok {
			if jd != v {
				logs.Info("ali的key:", k, "ali的size:", v, "JD的size:", jd)
			}
		} else {
			logs.Info("JDmap中沒有找到:", k)
			a.shrotmap[k] = v
		}
	}

	accessKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
	secretKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
	config := &aws.Config{
		Region:      aws.String("cn-south-1"),
		Endpoint:    aws.String("xxxxxxxxxxx"),
		DisableSSL:  aws.Bool(false),
		Credentials: creds,
	}
	newSession, err := session.NewSession(config)

	uploader := s3manager.NewUploader(newSession)
	fmt.Println("---------JDshortmap szie:", len(a.shrotmap))
	for k, _ := range a.shrotmap {
		closer, err := bucket.GetObject(k)
		if err != nil {
			logs.Error("alioss下載文件失敗:", err.Error())
			continue
		}
		_, err = uploader.Upload(&s3manager.UploadInput{
			Bucket: aws.String("xxxxxx"),
			Key:    aws.String(k),
			Body:   closer,
		})
		if err != nil {
			logs.Error("JD上傳失敗:", err.Error())
		} else {
			logs.Info("上傳成功 key:", k)
		}
		if closer != nil {
			closer.Close()
		}
	}
}

//創建新的AWS S3客戶端
func NewS3Client() *s3.S3 {
	accessKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
	secretKey := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
	config := &aws.Config{
		Region:      aws.String("cn-south-1"),
		Endpoint:    aws.String("xxxxxxxx"),
		DisableSSL:  aws.Bool(false),
		Credentials: creds,
	}
	client := s3.New(session.New(config))
	return client
}