initial commit

.Rbuildignore

@@ -0,0 +1,10 @@
+^.*\.Rproj$
+^\.Rproj\.user$
+^\.travis\.yml$
+^README\.*Rmd$
+^README\.*html$
+^NOTES\.*Rmd$
+^NOTES\.*html$
+^\.codecov\.yml$
+^README_files$
+^doc$

.codecov.yml

@@ -0,0 +1 @@
+comment: false

.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+.Rproj.user
+.Rhistory
+.RData
+.Rproj
+src/*.o
+src/*.so
+src/*.dll

.travis.yml

@@ -0,0 +1,31 @@
+language: r
+
+warnings_are_errors: true
+
+sudo: required
+
+cache: packages
+
+r:
+ - oldrel
+ - release
+ - devel
+
+apt_packages:
+  - libv8-dev
+  - xclip
+
+env:
+ global:
+   - CRAN: http://cran.rstudio.com
+
+after_success:
+  - Rscript -e 'covr::codecov()'
+
+notifications:
+  email:
+    - bob@rud.is
+  irc:
+    channels:
+      - "104.236.112.222#builds"
+    nick: travisci

DESCRIPTION

@@ -0,0 +1,22 @@
+Package: metis
+Type: Package
+Title: Helpers for Accessing and Querying Amazon Athena
+Version: 0.1.0
+Date: 2017-05-16
+Author: Bob Rudis (bob@rud.is)
+Maintainer: Bob Rudis <bob@rud.is>
+Description: Helpers for Accessing and Querying Amazon Athena. Including a lightweight RJDBC shim.
+URL: https://github.com/hrbrmstr/metis
+BugReports: https://github.com/hrbrmstr/metis/issues
+License: AGPL
+Suggests:
+    testthat,
+    covr
+Depends:
+    R (>= 3.2.0),
+    RJDBC
+Imports:
+    DBI,
+    dplyr,
+    ini
+RoxygenNote: 6.0.1

NAMESPACE

@@ -0,0 +1,14 @@
+# Generated by roxygen2: do not edit by hand
+
+export(Athena)
+export(athena_connect)
+exportClasses(AthenaConnection)
+exportClasses(AthenaDriver)
+exportClasses(AthenaResult)
+exportMethods(dbConnect)
+exportMethods(dbGetQuery)
+exportMethods(dbSendQuery)
+import(DBI)
+import(RJDBC)
+import(dplyr)
+import(ini)

NEWS.md

@@ -0,0 +1,2 @@
+0.1.0 
+* Initial release

R/jdbc.r

@@ -0,0 +1,91 @@
+#' AthenaJDBC
+#'
+#' @export
+setClass("AthenaDriver", representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef"))
+
+#' AthenaJDBC
+#'
+#' @export
+Athena <- function(identifier.quote='`') {
+  drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver",
+              system.file("AthenaJDBC41-1.0.1.jar", package="metis"),
+              identifier.quote="'")
+  return(as(drv, "AthenaDriver"))
+}
+
+#' AthenaJDBC
+#'
+#' @export
+setMethod(
+
+  "dbConnect",
+  "AthenaDriver",
+
+  def = function(drv,
+                 provider = "com.amazonaws.athena.jdbc.shaded.com.amazonaws.auth.EnvironmentVariableCredentialsProvider",
+                 conn_string = 'jdbc:awsathena://athena.us-east-1.amazonaws.com:443/',
+                 schema_name, ...) {
+
+    if (!is.null(provider)) {
+
+      jc <- callNextMethod(drv, conn_string,
+                           s3_staging_dir=Sys.getenv("AWS_S3_STAGING_DIR"),
+                           schema_name=schema_name,
+                           aws_credentials_provider_class=provider, ...)
+
+    } else {
+
+      jc <- callNextMethod(drv,
+                       'jdbc:awsathena://athena.us-east-1.amazonaws.com:443/',
+                       s3_staging_dir=Sys.getenv("AWS_S3_STAGING_DIR"),
+                       schema_name=schema_name,
+                       user = Sys.getenv("AWS_ACCESS_KEY_ID"),
+                       password = Sys.getenv("AWS_SECRET_ACCESS_KEY"))
+
+    }
+
+    return(as(jc, "AthenaConnection"))
+
+  }
+
+)
+
+#' AthenaJDBC
+#'
+#' @export
+setClass("AthenaConnection", contains = "JDBCConnection")
+
+#' AthenaJDBC
+#'
+#' @export
+setClass("AthenaResult", contains = "JDBCResult")
+
+#' AthenaJDBC
+#'
+#' @export
+setMethod(
+
+  "dbSendQuery",
+  "AthenaDriver",
+
+  def = function(conn, statement, ...) {
+    return(as(callNextMethod(), "AthenaResult"))
+  }
+
+)
+
+#' AthenaJDBC
+#'
+#' @export
+setMethod(
+
+  "dbGetQuery",
+  signature(conn="AthenaConnection", statement="character"),
+
+  def = function(conn, statement, ...) {
+    r <- dbSendQuery(conn, statement, ...)
+    on.exit(.jcall(r@stat, "V", "close"))
+    dplyr::tbl_df(fetch(r, -1, block=256))
+  }
+
+)

R/metis-package.R

@@ -0,0 +1,12 @@
+#' Helpers for Accessing and Querying Amazon Athena
+#'
+#' Including a lightweight RJDBC shim.
+#'
+#' @name metis
+#' @docType package
+#' @author Bob Rudis (bob@@rud.is)
+#' @import RJDBC
+#' @import DBI
+#' @import dplyr
+#' @import ini
+NULL

R/metis.r

@@ -0,0 +1,48 @@
+#' Make a JDBC connection to Athena
+#'
+#' Handles the up-front JDBC config
+#'
+#' For all connection types it is expected that you have the following environment variables
+#' defined (a good place is `~/.Renviron`):
+#'
+#' - `AWS_S3_STAGING_DIR`: the name of the S3 bucket where Athena can write stuff
+#' - `AWS_PROFILE`: the AWS profile ID in `~/.aws/credentials` (defaults to `default` if not present)
+#'
+#' For `simple` == `FALSE` the expectation is that you're working with a managed
+#' `~/.aws/credentials` file.
+#'
+#' @md
+#' @param default_schema def sch
+#' @param simple simple
+#' @export
+athena_connect <- function(default_schema, simple=FALSE) {
+
+  athena_jdbc <- Athena()
+
+  aws_config <- ini::read.ini(path.expand("~/.aws/credentials"))
+  aws_profile <- aws_config[Sys.getenv("AWS_PROFILE", "default")][[1]]
+
+  Sys.unsetenv("AWS_ACCESS_KEY_ID")
+  Sys.unsetenv("AWS_SECRET_ACCESS_KEY")
+
+  Sys.setenv(AWS_ACCESS_KEY_ID = aws_profile$aws_access_key_id)
+  Sys.setenv(AWS_SECRET_ACCESS_KEY = aws_profile$aws_secret_access_key)
+
+  con <- NULL
+
+  if (!simple) {
+
+    Sys.unsetenv("AWS_SESSION_TOKEN")
+    Sys.setenv(AWS_SESSION_TOKEN = aws_profile$aws_session_token)
+
+    con <- dbConnect(athena_jdbc, schema_name = default_schema)
+
+  } else {
+
+    con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema)
+
+  }
+
+  con
+
+}

README.Rmd

@@ -0,0 +1,63 @@
+---
+output: rmarkdown::github_document
+---
+
+[`metis`](https://en.wikipedia.org/wiki/Metis_(mythology)) : Helpers for Accessing and Querying Amazon Athena
+
+Including a lightweight RJDBC shim.
+
+![](https://upload.wikimedia.org/wikipedia/commons/thumb/5/53/Winged_goddess_Louvre_F32.jpg/300px-Winged_goddess_Louvre_F32.jpg)
+
+THIS IS SUPER ALPHA QUALITY. NOTHING TO SEE HERE. MOVE ALONG.
+
+The goal will be to get around enough of the "gotchas" that are preventing raw RJDBC Athena
+connecitons from "just working" with `dplyr` v0.6.0+ and also get around the [`fetchSize` problem](https://www.reddit.com/r/aws/comments/6aq22b/fetchsize_limit/) without having to not use `dbGetQuery()`.
+
+It will also support more than the vanilla id/secret auth mechism (it currently support the default basic auth and temp token auth, the latter via environment variables).
+
+See the **Usage** section for an example.
+
+The following functions are implemented:
+
+- `athena_connect`:	Make a JDBC connection to Athena (this returns an `AthenaConnection` object which is a super-class of it's RJDBC vanilla counterpart)
+- `Athena`:	AthenaJDBC`
+- `AthenaConnection-class`:	AthenaJDBC
+- `AthenaDriver-class`:	AthenaJDBC
+- `AthenaResult-class`:	AthenaJDBC
+- `dbConnect-method`:	AthenaJDBC
+- `dbGetQuery-method`:	AthenaJDBC
+- `dbSendQuery-method`:	AthenaJDBC
+
+### Installation
+
+```{r eval=FALSE}
+devtools::install_github("hrbrmstr/metis")
+```
+
+```{r message=FALSE, warning=FALSE, error=FALSE, include=FALSE}
+options(width=120)
+```
+
+### Usage
+
+```{r message=FALSE, warning=FALSE, error=FALSE}
+library(metis)
+library(dplyr)
+
+# current verison
+packageVersion("metis")
+```
+
+```{r message=FALSE, warning=FALSE, error=FALSE, eval=FALSE}
+ath <- athena_connect("your_schema_name")
+
+res <- dbGetQuery(ath, "
+SELECT format_datetime(timestamp, 'yyyy-MM-dd HH:00:00') timestamp,
+        port as field, count(port) cnt_field FROM your_schema_name.your_table_name
+        WHERE CONTAINS(ARRAY['201705'], date)
+        AND port IN (445, 139, 3389)
+        AND timestamp > date '2017-05-01'
+        AND timestamp <= date '2017-05-22'
+GROUP BY format_datetime(timestamp, 'yyyy-MM-dd HH:00:00'), port LIMIT 1000000
+")
+```

README.md

@@ -0,0 +1,57 @@
+
+[`metis`](https://en.wikipedia.org/wiki/Metis_(mythology)) : Helpers for Accessing and Querying Amazon Athena
+
+Including a lightweight RJDBC shim.
+
+![](https://upload.wikimedia.org/wikipedia/commons/thumb/5/53/Winged_goddess_Louvre_F32.jpg/300px-Winged_goddess_Louvre_F32.jpg)
+
+THIS IS SUPER ALPHA QUALITY. NOTHING TO SEE HERE. MOVE ALONG.
+
+The goal will be to get around enough of the "gotchas" that are preventing raw RJDBC Athena connecitons from "just working" with `dplyr` v0.6.0+ and also get around the [`fetchSize` problem](https://www.reddit.com/r/aws/comments/6aq22b/fetchsize_limit/) without having to not use `dbGetQuery()`.
+
+It will also support more than the vanilla id/secret auth mechism (it currently support the default basic auth and temp token auth, the latter via environment variables).
+
+See the **Usage** section for an example.
+
+The following functions are implemented:
+
+-   `athena_connect`: Make a JDBC connection to Athena (this returns an `AthenaConnection` object which is a super-class of it's RJDBC vanilla counterpart)
+-   `Athena`: AthenaJDBC\`
+-   `AthenaConnection-class`: AthenaJDBC
+-   `AthenaDriver-class`: AthenaJDBC
+-   `AthenaResult-class`: AthenaJDBC
+-   `dbConnect-method`: AthenaJDBC
+-   `dbGetQuery-method`: AthenaJDBC
+-   `dbSendQuery-method`: AthenaJDBC
+
+### Installation
+
+``` r
+devtools::install_github("hrbrmstr/metis")
+```
+
+### Usage
+
+``` r
+library(metis)
+library(dplyr)
+
+# current verison
+packageVersion("metis")
+```
+
+    ## [1] '0.1.0'
+
+``` r
+ath <- athena_connect("your_schema_name")
+
+res <- dbGetQuery(ath, "
+SELECT format_datetime(timestamp, 'yyyy-MM-dd HH:00:00') timestamp,
+        port as field, count(port) cnt_field FROM your_schema_name.your_table_name
+        WHERE CONTAINS(ARRAY['201705'], date)
+        AND port IN (445, 139, 3389)
+        AND timestamp > date '2017-05-01'
+        AND timestamp <= date '2017-05-22'
+GROUP BY format_datetime(timestamp, 'yyyy-MM-dd HH:00:00'), port LIMIT 1000000
+")
+```

man/Athena.Rd

@@ -0,0 +1,11 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\name{Athena}
+\alias{Athena}
+\title{AthenaJDBC}
+\usage{
+Athena(identifier.quote = "`")
+}
+\description{
+AthenaJDBC
+}

man/AthenaConnection-class.Rd

@@ -0,0 +1,9 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{class}
+\name{AthenaConnection-class}
+\alias{AthenaConnection-class}
+\title{AthenaJDBC}
+\description{
+AthenaJDBC
+}

man/AthenaDriver-class.Rd

@@ -0,0 +1,9 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{class}
+\name{AthenaDriver-class}
+\alias{AthenaDriver-class}
+\title{AthenaJDBC}
+\description{
+AthenaJDBC
+}

man/AthenaResult-class.Rd

@@ -0,0 +1,9 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{class}
+\name{AthenaResult-class}
+\alias{AthenaResult-class}
+\title{AthenaJDBC}
+\description{
+AthenaJDBC
+}

man/athena_connect.Rd

@@ -0,0 +1,27 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/metis.r
+\name{athena_connect}
+\alias{athena_connect}
+\title{Make a JDBC connection to Athena}
+\usage{
+athena_connect(default_schema, simple = FALSE)
+}
+\arguments{
+\item{default_schema}{def sch}
+
+\item{simple}{simple}
+}
+\description{
+Handles the up-front JDBC config
+}
+\details{
+For all connection types it is expected that you have the following environment variables
+defined (a good place is \code{~/.Renviron}):
+\itemize{
+\item \code{AWS_S3_STAGING_DIR}: the name of the S3 bucket where Athena can write stuff
+\item \code{AWS_PROFILE}: the AWS profile ID in \code{~/.aws/credentials} (defaults to \code{default} if not present)
+}
+
+For \code{simple} == \code{FALSE} the expectation is that you're working with a managed
+\code{~/.aws/credentials} file.
+}

man/dbConnect-AthenaDriver-method.Rd

@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{methods}
+\name{dbConnect,AthenaDriver-method}
+\alias{dbConnect,AthenaDriver-method}
+\title{AthenaJDBC}
+\usage{
+\S4method{dbConnect}{AthenaDriver}(drv,
+  provider = "com.amazonaws.athena.jdbc.shaded.com.amazonaws.auth.EnvironmentVariableCredentialsProvider",
+  conn_string = "jdbc:awsathena://athena.us-east-1.amazonaws.com:443/",
+  schema_name, ...)
+}
+\description{
+AthenaJDBC
+}

man/dbGetQuery-AthenaConnection-character-method.Rd

@@ -0,0 +1,12 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{methods}
+\name{dbGetQuery,AthenaConnection,character-method}
+\alias{dbGetQuery,AthenaConnection,character-method}
+\title{AthenaJDBC}
+\usage{
+\S4method{dbGetQuery}{AthenaConnection,character}(conn, statement, ...)
+}
+\description{
+AthenaJDBC
+}

man/dbSendQuery-AthenaDriver-ANY-method.Rd

@@ -0,0 +1,12 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/jdbc.r
+\docType{methods}
+\name{dbSendQuery,AthenaDriver,ANY-method}
+\alias{dbSendQuery,AthenaDriver,ANY-method}
+\title{AthenaJDBC}
+\usage{
+\S4method{dbSendQuery}{AthenaDriver,ANY}(conn, statement, ...)
+}
+\description{
+AthenaJDBC
+}

man/metis.Rd

@@ -0,0 +1,13 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/metis-package.R
+\docType{package}
+\name{metis}
+\alias{metis}
+\alias{metis-package}
+\title{Helpers for Accessing and Querying Amazon Athena}
+\description{
+Including a lightweight RJDBC shim.
+}
+\author{
+Bob Rudis (bob@rud.is)
+}

metis.Rproj

@@ -0,0 +1,21 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
+
+StripTrailingWhitespace: Yes
+
+BuildType: Package
+PackageUseDevtools: Yes
+PackageInstallArgs: --no-multiarch --with-keep.source
+PackageBuildArgs: --resave-data
+PackageRoxygenize: rd,collate,namespace

tests/test-all.R

@@ -0,0 +1,2 @@
+library(testthat)
+test_check("metis")

tests/testthat/test-metis.R

@@ -0,0 +1,6 @@
+context("basic functionality")
+test_that("we can do something", {
+
+  #expect_that(some_function(), is_a("data.frame"))
+
+})