scala-s3
aws s3 client in scala
git clone https://9o.is/git/scala-s3.git
commit d1f2e5bdd7b089a3cc66b15b977f98496e9e211e parent bcc2cac6bbf887bebe3f01dc51b10002edc1acb6 Author: Jul <jul@9o.is> Date: Fri, 6 Jun 2014 01:55:38 -0400 implemented Diffstat:
| M | README.md | | | 53 | ++++++++++++++++++++++++++++++++++++++++++++++++++--- |
| M | project/Build.scala | | | 2 | +- |
| A | src/main/scala/inc/pyc/aws/s3/AmazonS3.scala | | | 177 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/scala/inc/pyc/aws/s3/Bucket.scala | | | 11 | +++++++++++ |
| A | src/main/scala/inc/pyc/aws/s3/S3.scala | | | 45 | +++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/test/scala/inc/pyc/aws/s3/S3Spec.scala | | | 91 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
6 files changed, 375 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md @@ -1,4 +1,51 @@ -SBT Starter Template -==================== +Amazon S3 module +============= -Use this to start any project that requires sbt. +The `aws-s3` module provides basic support for interacting with Amazon's +S3 service by providing additional handlers which sign the HTTP +request in accordance with the [S3 authentication specifications][1]. +It's possible both to sign requests for Authorization Header +Authentication and to generate signed request URIs that can be handed +out to third parties for Query String Authentication. +The module also provides a convenience class for interacting with S3 +Buckets. + +## Usage ## +To use the `aws-s3` module, you will need to first sign up for the S3 +service. Afterwards, Amazon should provide you with an AWS Access Key +and an AWS Secret Access Key. You will need both of these to be able +to use the module. + +Below is an example of retrieving a file from S3: + + import dispatch._ + import inc.pyc.aws.s3._ + + val bucketName = "bucket-name" + val access_key = "XXXXXXXXX" + val secret_key = "XXXXXXXXX" + + val s3 = S3(access_key, secret_key, bucketName, "path/in/bucket") + + s3.createFile(testFile.getName, testFile, "text/plain")() map { + response => + // Handle response here + } + +## Testing + +To test the module, you'll need to set 3 system properties: + +* awsBucket +* awsAccessKey +* awsSecretAccessKey + +When using sbt 0.11, you can do the following: + + eval System.setProperty("awsBucket", "bucket-name") + eval System.setProperty("awsAccessKey", "XXXXXXXXX") + eval System.setProperty("awsSecretAccessKey", "XXXXXXXXX") + +After that, you can just run `test` and all of the tests should pass. + +[1]: http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html diff --git a/project/Build.scala b/project/Build.scala @@ -11,7 +11,7 @@ object LiftProjectBuild extends Build { Seq( "net.databinder.dispatch" %% "dispatch-core" % "0.10.0", "ch.qos.logback" % "logback-classic" % "1.0.13" % "compile", - "org.scalatest" %% "scalatest" % "1.9.2" % "test" + "org.scala-tools.testing" %% "specs" % "1.6.9" ) ) } diff --git a/src/main/scala/inc/pyc/aws/s3/AmazonS3.scala b/src/main/scala/inc/pyc/aws/s3/AmazonS3.scala @@ -0,0 +1,176 @@ +package inc.pyc +package aws +package s3 + +import dispatch._ +import com.ning.http._ +import client._ +import util._ + +import java.net.URLEncoder._ +import java.util.{Date, Locale, SimpleTimeZone} +import java.text.SimpleDateFormat +import javax.crypto + +/** + * Handles the signing, authentication and requests for S3. + */ +private[s3] object AmazonS3 { + + /** + * S3 root domain + */ + lazy val Root = "s3.amazonaws.com" + + /** + * Date Format: http://www.ietf.org/rfc/rfc0822.txt + */ + object rfc822DateParser extends SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US) { + this.setTimeZone(new SimpleTimeZone(0, "GMT")) + } + + def trim(s: String): String = s.dropWhile(_ == ' ').reverse.dropWhile(_ == ' ').reverse.toString + + /** + * MD5 hash of bytes. + */ + def md5(bytes: Array[Byte]) = { + import java.security.MessageDigest + + val r = MessageDigest.getInstance("MD5") + r.reset() + r.update(bytes) + Base64.encode(r.digest) + } + + /** + * MD5 hash of InputStream. + */ + def md5(stream: java.io.InputStream) = { + import java.security.MessageDigest + + val buffer = new Array[Byte](1024) + val r = MessageDigest.getInstance("MD5") + var numRead: Int = 0 + do { + numRead = stream.read(buffer) + if (numRead > 0) { + r.update(buffer, 0, numRead) + } + } while (numRead != -1) + + stream.close() + Base64.encode(r.digest()) + } + + /** + * Signing of information to be transfered to S3 servers with the secret key. + */ + def sign(method: String, path: String, secretKey: String, date: Date, + contentType: Option[String], contentMd5: Option[String], amzHeaders: Map[String, Set[String]]): String = { + sign(method, path, secretKey, Left(date), contentType, contentMd5, amzHeaders) + } + + /** + * Signing of information to be transfered to S3 servers with the secret key. + */ + def sign(method: String, path: String, secretKey: String, dateOrExpires: Either[Date, Long], + contentType: Option[String], contentMd5: Option[String], amzHeaders: Map[String, Set[String]]) = { + val SHA1 = "HmacSHA1" + val message = canonicalString(method, path, dateOrExpires, contentType, contentMd5, amzHeaders) + val sig = { + val mac = crypto.Mac.getInstance(SHA1) + val key = new crypto.spec.SecretKeySpec(bytes(secretKey), SHA1) + mac.init(key) + Base64.encode(mac.doFinal(bytes(message))) + } + sig + } + + def signedUri(accessKey: String, secretKey: String, method: String, path: String, amzHeaders: Map[String, Set[String]], + expires: Long = defaultExpiryTime, + contentType: Option[String] = None, contentMd5: Option[String] = None) = { + val signed = encode(sign(method, path, secretKey, Right(expires), contentType, contentMd5, amzHeaders), "UTF-8") + "%s?Signature=%s&Expires=%s&AWSAccessKeyId=%s".format(path, signed, expires, accessKey) + } + + def defaultExpiryTime = System.currentTimeMillis() / 1000 + 600 + + /** + * @return the canonical request string that needs to be signed for authentication + */ + def canonicalString(method: String, path: String, dateOrExpires: Either[Date, Long], contentType: Option[String], + contentMd5: Option[String], amzHeaders: Map[String, Set[String]]) = { + val amzString = amzHeaders.toList.sortWith(_._1.toLowerCase < _._1.toLowerCase).map { + case (k, v) => "%s:%s".format(k.toLowerCase, v.map(trim).mkString(",")) + } + val dateExpiresString = dateOrExpires match { + case Left(date) => rfc822DateParser.format(date) + case Right(expires) => expires.toString + } + (method :: contentMd5.getOrElse("") :: contentType.getOrElse("") :: dateExpiresString :: Nil) ++ amzString ++ List(path) mkString "\n" + } + + /** + * Bytes of string in UTF-8 encoding. + */ + def bytes(s: String) = s.getBytes("UTF-8") + + + implicit def Request2S3RequestSigner(r: RequestBuilder) = new S3RequestSigner(r) + implicit def Request2S3RequestSigner(r: String) = new S3RequestSigner(new RequestBuilder().setUrl(r)) + + /** + * Signing of S3 requests. + */ + class S3RequestSigner(r: RequestBuilder) { + + import scala.collection.JavaConverters._ + + protected def path = RawUri(r.build.getUrl).path.getOrElse("") + + /** + * Handler for authentication information, access and secret key, for signing. + */ + def <@(accessKey: String, secretKey: String) = { + val req = r.build + val contentStream = req.getStreamData + val contentMd5 = if (req.getContentLength <= 0) None else Some(md5(contentStream)) + + for (cmd5 <- contentMd5) + r.addHeader("Content-MD5", cmd5) + + val headers = req.getHeaders + val contentType = headers.keySet.asScala.find { + _.toLowerCase == "content-type" + }.map { + headers.get(_).asScala.head + } + + val d = new Date + r.addHeader("Authorization", "AWS %s:%s".format(accessKey, sign(req.getMethod, path, secretKey, d, contentType, contentMd5, amazonHeaders))) + r.addHeader("Date", rfc822DateParser.format(d)) + r + } + + def signed(accessKey: String, secretKey: String, expires: Long = defaultExpiryTime, + contentType: Option[String] = None, contentMd5: Option[String] = None): RequestBuilder = { + val req = r.build + val path = RawUri(req.getUrl).path.getOrElse("") + val uri = signedUri(accessKey, secretKey, req.getMethod, path, amazonHeaders, expires, contentType, contentMd5) + val requestHeaders = for { + (key, Some(value)) <- Map("Content-Type" -> contentType, "Content-Md5" -> contentMd5) + } yield key -> value + (:/(Root) / uri.substring(1)).setMethod(req.getMethod).setHeaders(req.getHeaders).secure <:< requestHeaders + } + + private def amazonHeaders = { + val headers = r.build.getHeaders + headers.keySet.asScala.filter { + _.toLowerCase.startsWith("x-amz") + } + .map(name => name -> headers.get(name).asScala.toSet).toMap + } + } + +} +\ No newline at end of file diff --git a/src/main/scala/inc/pyc/aws/s3/Bucket.scala b/src/main/scala/inc/pyc/aws/s3/Bucket.scala @@ -0,0 +1,10 @@ +package inc.pyc +package aws +package s3 + +import dispatch._ +import com.ning.http.client.RequestBuilder + +private object Bucket extends (String => RequestBuilder) { + def apply(name: String) = :/(AmazonS3.Root) / name +} +\ No newline at end of file diff --git a/src/main/scala/inc/pyc/aws/s3/S3.scala b/src/main/scala/inc/pyc/aws/s3/S3.scala @@ -0,0 +1,44 @@ +package inc.pyc +package aws +package s3 + +import dispatch._ +import Defaults._ +import java.io._ +import com.ning.http.client.Response + +/** + * Amazon S3 API + */ +case class S3(access_key: String, secret_key: String, bucket: String, path: String = "") { + import AmazonS3._ + + /** + * Creates a file given a file name, binary data and file content-type. + */ + def createFile(fn: String, data: Array[Byte], contentType: String): Future[Response] = + Http.configure(_ setCompressionEnabled true)(Bucket(bucket).PUT.setBody(data) / (path + fn) <:< + Map("content-type" -> contentType) <@(access_key, secret_key)) + + /** + * Creates a file given a file name, a file and file content-type. + */ + def createFile(fn: String, file: java.io.File, contentType: String): Future[Response] = { + val bis = new BufferedInputStream(new FileInputStream(file)) + val bArray = Stream.continually(bis.read).takeWhile(-1 !=).map(_.toByte).toArray + + createFile(fn, bArray, contentType) + } + + /** + * Deletes a file given a file name. + */ + def deleteFile(fn: String) = + Http.configure(_ setCompressionEnabled true)(Bucket(bucket).DELETE / + (path + fn) <@(access_key, secret_key)) + + /** + * Generates file's S3 url given a file name. + */ + def fileUrl(fn: String) = "http://" + AmazonS3.Root + "/" + bucket + path + fn +} +\ No newline at end of file diff --git a/src/test/scala/inc/pyc/aws/s3/S3Spec.scala b/src/test/scala/inc/pyc/aws/s3/S3Spec.scala @@ -0,0 +1,91 @@ +package inc.pyc +package aws +package s3 + +import AmazonS3._ +import java.util.{TimeZone, Calendar} +import org.specs._ +import dispatch._ +import java.io.{File,FileWriter} +import com.ning.http.client.Response + +object S3Spec extends Specification { + + val bucketName = getValue("awsBucket") + val access_key = getValue("awsAccessKey") + val secret_key = getValue("awsSecretAccessKey") + + def s3(): Option[S3] = + for { + bucketName <- bucketName + access_key <- access_key + secret_key <- secret_key + } yield S3(access_key, secret_key, bucketName) + + def shouldWeSkip_? = List(s3) must notContain(None).orSkip + + def newTempFile = { + val testFile = File.createTempFile("s3specs","bin") + val writer = new FileWriter(testFile) + writer.write("testing") + writer.close + testFile + } + + "S3" should { + "be able to create a file" in { + shouldWeSkip_? + + val testFile = newTempFile + s3() map { + s3 => + val r: Response = s3.createFile(testFile.getName, testFile, "text/plain")() + r.getStatusCode must be_==(200) + } + } + + "be able to delete a file" in { + shouldWeSkip_? + + val testFile = newTempFile + s3() map { + s3 => + val r: Response = s3.deleteFile(testFile.getName)() + r.getStatusCode must be_==(200) + } + } + + "create the correct canonical string for a request with a date header" in { + val expected = "PUT\nmd5sum\ntext/plain\nMon, 17 Oct 2011 11:43:29 GMT\nx-amz-meta-author:john@doe.com\n/mybucket/newobject123" + val cal = Calendar.getInstance(TimeZone.getTimeZone("GMT")) + cal.set(2011, 9, 17, 11, 43, 29) + val date = cal.getTime() + val amzHeaders = Map("x-amz-meta-author" -> Set("john@doe.com")) + val res = canonicalString("PUT", "/mybucket/newobject123", Left(date), Some("text/plain"), Some("md5sum"), amzHeaders) + res must_== expected + } + + "create the correct canonical string for a request with an Expires value" in { + val expires = System.currentTimeMillis() / 1000 + 30 + val expected = "PUT\n\n\n%s\nx-amz-meta-author:john@doe.com\n/mybucket/newobject123".format(expires.toString) + val amzHeaders = Map("x-amz-meta-author" -> Set("john@doe.com")) + val res = canonicalString("PUT", "/mybucket/newobject123", Right(expires), None, None, amzHeaders) + res must_== expected + } + + } + + doAfterSpec { + Http.shutdown() + } + + def getValue(key: String): Option[String] = { + if (System.getenv(key) != null) { + Some(System.getenv(key)) + } else if (System.getProperty(key) != null) { + Some(System.getProperty(key)) + } else { + None + } + } +}