aws sdk之nodejs s3

upload 和putObject有什么区别，同样是上传或新增一个object
upload适用于比较大的文件，putObject适用于小的文件内容，upload支持自定义多线程并发上传

var AWS = require('aws-sdk');
AWS.config.loadFromPath('./config/aws.config');
var fs = require('fs');
//var s3 = new AWS.S3({params:{Bucket:'xxx'}});

// OK
// s3.listBuckets(function(error, data) {
  // if (error) {
    // console.log(error); // error is Response.error
  // } else {
    // console.log(data); // data is Response.data
  // }
// });

// s3.getObject({Key:'download_test.log'}).on('success', function(response) {
  // console.log("Key was", response.request.params.Key);
// }).send();

// var s3 = new AWS.S3();
// var params = {Bucket: 'mochong', Key:'message_center.txt'};
// var file = require('fs').createWriteStream('./deploy_local.txt');
// s3.getObject(params).createReadStream().pipe(file);

/** create bucket 名字不能包含`_`等字符，必须唯一**/
/**
var s3 = new AWS.S3();
var params = {
  Bucket: 'mochongmessagecenter', //required 
  ACL: 'private',
  CreateBucketConfiguration: {
    LocationConstraint: 'us-west-2'
  }
};
s3.createBucket(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});
**/

/**delete bucket**/
/**
 *
var s3 = new AWS.S3();
var params = {
  Bucket: 'mochongmessagecenter' //required
};
s3.deleteBucket(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});
**/

/** init  multipart upload  开始多线程上传，获取uploadID**/
/**
var params = {
  Bucket: 'mochong', //required
  Key: 'upload.txt', // required
  ACL: 'private',
 // CacheControl: 'STRING_VALUE',
 // ContentDisposition: 'STRING_VALUE',
  ContentEncoding: 'utf8',
  ContentLanguage: 'zh-cn',
  ContentType: 'plain',
  //Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
  //GrantFullControl: 'STRING_VALUE',
 // GrantRead: 'STRING_VALUE',
  //GrantReadACP: 'STRING_VALUE',
  //GrantWriteACP: 'STRING_VALUE',
  //Metadata: {
  //  someKey: 'STRING_VALUE',
    // anotherKey: ...
  //},
  //RequestPayer: 'requester',
  //SSECustomerAlgorithm: 'STRING_VALUE',
  //SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
 // SSECustomerKeyMD5: 'STRING_VALUE',
  //SSEKMSKeyId: 'STRING_VALUE',
  //ServerSideEncryption: 'AES256 | aws:kms',
  //StorageClass: 'STANDARD | REDUCED_REDUNDANCY | STANDARD_IA',
  //WebsiteRedirectLocation: 'STRING_VALUE'
};
var s3 = new AWS.S3();
// { 
	// Bucket: 'mochong',
	// Key: 'upload.txt',
	// UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB' 
// }

s3.createMultipartUpload(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});

**/

/**开始多线程上传**/
// var s3 = new AWS.S3();
// fs.readFile('./download_test.log', 'utf8', function (err,data) {
  // if (err) {
    // return console.log(err);
  // }
  // console.log(data);
  // var params = {
	  // Bucket: 'mochong', // required 
	  // Key: 'upload.txt', // required 
	  // PartNumber: 5, // 1~10000
	  // UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', /* required */
	  // Body: data,
	  // ContentLength: Buffer.byteLength(data, 'utf8'),
	  // //ContentMD5: 'STRING_VALUE',
	  // //RequestPayer: 'requester',
	 // // SSECustomerAlgorithm: 'STRING_VALUE',
	 // // SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
	  // //SSECustomerKeyMD5: 'STRING_VALUE'
	// };
  // s3.uploadPart(params, function(err, data) {
	  // if (err) console.log(err, err.stack); // an error occurred
	  // else     console.log(data);           // successful response { ETag: '"6f9769150b95562cd1831f82e7026338"' }
  // });
// });


/**结束上传**/
/*
var s3 = new AWS.S3();
var params = {
  Bucket: 'mochong', // required
  Key: 'upload.txt', // required 
  UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', // required
  MultipartUpload: {
    Parts: [
      {
        ETag: '6f9769150b95562cd1831f82e7026338',
        PartNumber: 5 // must >= 1
      },
    ]
  },
  RequestPayer: 'requester'
};
s3.completeMultipartUpload(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
  // response data: 
  //{ Location: 'https://mochong.s3-us-west-2.amazonaws.com/upload.txt',
  // Bucket: 'mochong',
  // Key: 'upload.txt',
  // ETag: '"0598c44b0e25dde9e8dfdb55043b3a5f-1"' }

});*/



/**忽略上传**/
/*
var s3 = new AWS.S3();
var params = {
  Bucket: 'mochong', //required 
  Key: 'upload.txt', //required 
  UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', //required 
  RequestPayer: 'requester'
};
s3.abortMultipartUpload(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response: {}
});*/

/**智能上传 要注意必须用stream，buffer的类型**/
//var s3 = new AWS.S3();

// 不要用readFile.因为 s3的upload只支持stream，buffer,blob
//Uploads an arbitrarily sized buffer, blob, or stream, 
//using intelligent concurrent handling of parts if the payload is large enough. 
//You can configure the concurrent queue size by setting options.
// Note that this is the only operation for which the SDK can retry requests with stream bodies.

/*
var list = [];
var fileReadStream = fs.createReadStream('./log/api.log', {encoding: 'utf8'});
var tmpBuffer;
fileReadStream.on('data', function (chunk) {
	tmpBuffer = new Buffer(chunk);
    list.push(tmpBuffer);
}).on('end', function () {
  fileReadStream.destroy();
  var stream = Buffer.concat(list);
  var params = {Bucket: 'mochong', Key: 'api.log', Body: stream};
    //var options = {partSize: 1 * 1024 * 1024, queueSize: 1}; //queueSize:骞跺彂鏁帮紝partSize锛氬崟鍧椾笂浼犲ぇ灏�,must >=5242880
  var upload = s3.upload(params, function(err, data) {
  		if (err) {
  			console.log(err);
  		}
	});
	
	upload.send(function (err, data) {
		stream.fill();
		if (err) console.log("Error:", err.message);
		else console.log(data);  
		
		// { ETag: '"3c6ecbcf8d6b65ccd16b47f603bff1b2"',
				  // Location: 'https://mochong.s3-us-west-2.amazonaws.com/download_test.log',
				  // key: 'download_test.log',
				  // Key: 'download_test.log',
				  // Bucket: 'mochong' 
				// }
	});
}).on('close', function (err) {
	console.log('Stream has been destroyed and file has been closed');
	tmpBuffer.fill();
});*/

/** list objectV2
 *Returns some or all (up to 1000) of the objects in a bucket. 
 * You can use the request parameters as selection criteria to return a subset of the objects in a bucket. 
 * Note: ListObjectsV2 is the revised List Objects API and we recommend you use this revised API for new application development.
 *  **/
/*
var params = {
  Bucket: 'mochong', // required 
  Delimiter: '|',
  EncodingType: 'url',
  //Marker: 'STRING_VALUE',
 // MaxKeys: 0,
 // Prefix: 'STRING_VALUE'
  Prefix: 'message_center/',
  StartAfter: 'STRING_VALUE'//StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this specified key. StartAfter can be any key in the bucket
};
var s3 = new AWS.S3();
s3.listObjectsV2(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});*/

/**
 *response data:
 *{ IsTruncated: false,
  Marker: 'STRING_VALUE',
  Contents: [],
  Name: 'mochong',
  Prefix: 'STRING_VALUE',
  Delimiter: '%7C',
  MaxKeys: 0,
  CommonPrefixes: [],
  EncodingType: 'url' }
 */


/** get object**/
/*
var s3 = new AWS.S3();
var params = {
  Bucket: 'mochong', //required 
  Key: 'message_center/deploy_local.txt', // required 
  //IfMatch: 'STRING_VALUE',// (String) Return the object only if its entity tag (ETag) is the same as the one specified, otherwise return a 412 (precondition failed).
  //IfModifiedSince: // new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
  //IfNoneMatch: 'STRING_VALUE',//(String) Return the object only if its entity tag (ETag) is different from the one specified, otherwise return a 304 (not modified).
  IfUnmodifiedSince:1471536000,// new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
  //Range: 'STRING_VALUE',
  //RequestPayer: 'requester',
  //ResponseCacheControl: 'STRING_VALUE',
  //ResponseContentDisposition: 'STRING_VALUE',
  //ResponseContentEncoding: 'STRING_VALUE',
  //ResponseContentLanguage: 'STRING_VALUE',
  //ResponseContentType: 'STRING_VALUE',
  //ResponseExpires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
  //SSECustomerAlgorithm: 'STRING_VALUE',
  //SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
  //SSECustomerKeyMD5: 'STRING_VALUE',
  //VersionId: 'STRING_VALUE'
};

s3.getObject(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});

*/


/*
success:
{ AcceptRanges: 'bytes',
  LastModified: 'Thu, 18 Aug 2016 06:49:12 GMT',
  ContentLength: '0',
  ETag: '"d41d8cd98f00b204e9800998ecf8427e"',
  ContentType: 'text/plain',
  Metadata: {},
  Body: <Buffer > }

error:
{
	[PreconditionFailed : At least one of the pre - conditions you specified did not hold]
	message : 'At least one of the pre-conditions you specified did not hold',
	code : 'PreconditionFailed',
	region : null,
	time : Fri Aug 19 2016 10 : 06 : 38 GMT + 0800(CST),
	requestId : 'D57010FB7BAD5DD3',
	extendedRequestId : 'GrAFqim56ytBONLN7oTj+Idhui6UlK66UfJ1GuP5X/aNocDpoufpOrRCdaxct3uUNXA/MBHIQIQ=',
	cfId : undefined,
	statusCode : 412,
	retryable : false,
	retryDelay : 38.56445546261966
}
'PreconditionFailed: At least one of the pre-conditions you specified did not hold\n    at Request.extractError (/home/web/message_center/node_modules/aws-sdk/lib/services/s3.js:538:35)\n    at Request.callListeners (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:105:20)\n    at Request.emit (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:77:10)\n    at Request.emit (/home/web/message_center/node_modules/aws-sdk/lib/request.js:661:14)\n    at Request.transition (/home/web/message_center/node_modules/aws-sdk/lib/request.js:22:10)\n    at AcceptorStateMachine.runTo (/home/web/message_center/node_modules/aws-sdk/lib/state_machine.js:14:12)\n    at /home/web/message_center/node_modules/aws-sdk/lib/state_machine.js:26:10\n    at Request.<anonymous> (/home/web/message_center/node_modules/aws-sdk/lib/request.js:38:9)\n    at Request.<anonymous> (/home/web/message_center/node_modules/aws-sdk/lib/request.js:663:12)\n    at Request.callListeners (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:115:18)'
*/

/*
{ AcceptRanges: 'bytes',
  LastModified: 'Thu, 18 Aug 2016 07:22:22 GMT',
  ContentLength: '23',
  ETag: '"0598c44b0e25dde9e8dfdb55043b3a5f-1"',
  ContentEncoding: 'utf-8',
  ContentLanguage: 'zh-cn',
  ContentType: 'plain',
  Metadata: {},
  Body: <Buffer 54 68 69 73 20 69 73 20 61 20 66 69 6c 65 20 74 6f 20 74 65 73 74 2e> }
*/



/** put object
 * @info Adds an object to a bucket.  
 ***/
var s3 = new AWS.S3();
var crypto = require('crypto');
var body = 'pull by put object api2';
var md5 = crypto.createHash('md5');
var contentMd5 = md5.update(body).digest('base64');
var params = {
  Bucket: 'mochong', //required 
  Key: 'new_folder/file_put_object.txt', // required 
  ACL: 'private',
  Body: body,
  //CacheControl: 'STRING_VALUE',
  //ContentDisposition: 'STRING_VALUE',
  ContentEncoding: 'utf8',
  ContentLanguage: 'zh-cn',
  ContentLength: body.length,
  ContentMD5: contentMd5,
  ContentType: 'text/plain',
  //Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
  //GrantFullControl: 'STRING_VALUE',
  //GrantRead: 'STRING_VALUE',
  //GrantReadACP: 'STRING_VALUE',
  //GrantWriteACP: 'STRING_VALUE',
  Metadata: {
    'x-amz-meta-source-type': 'server',// 自定义的metadata必须用x-amz-meta-做前缀
    'x-amz-meta-author': 'mochong'
    /* anotherKey: ... */
  },
  //RequestPayer: 'requester',
  //SSECustomerAlgorithm: 'STRING_VALUE',
  //SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
  //SSECustomerKeyMD5: 'STRING_VALUE',
  //SSEKMSKeyId: 'STRING_VALUE',
 // ServerSideEncryption: 'AES256 | aws:kms',
  StorageClass: 'STANDARD',
  //WebsiteRedirectLocation: 'STRING_VALUE'
};
s3.putObject(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else     console.log(data);           // successful response
});

/**
 * response:
 *{ ETag: '"e615fa17312c3e6e8a1682a5ce964d21"' } 
 */

function syncFile(file) {
	var bodyStream = fs.createReadStream(file.absolute_path, {encoding: 'utf8'});
    var options = {
        Bucket    : config.aws.bucket,
        Key    : config.aws.key_prefix + "/" + file.relative_path, 
        ContentLength : file.size,
        Body          : bodyStream
    };

    s3.putObject(options, function(err, data) {
    	if (err) {
	  		console.log("upload err,filepath:" + file.absolute_path);
	  		console.log(err);
	  		process.exit();
	  		return false;
	  	}
	  	bodyStream.destroy();
    });
}

報錯：
2016-08-22 13:01:52 [ERROR] sync logs failed,err:BadRequest: An error occurred when parsing the HTTP request., file:/home/web/xxxx,file count:160

如果一個文件正在上傳，又重複執行了上傳操作，就會報上面這個錯誤。