Browse Source

passed checks

boB Rudis 9 months ago
parent
commit
15a5352dc1
No known key found for this signature in database

+ 2
- 1
.Rbuildignore View File

@@ -8,5 +8,6 @@
8 8
 ^NOTES\.*html$
9 9
 ^\.codecov\.yml$
10 10
 ^README_files$
11
-^doc$
11
+^docs$
12 12
 ^CONDUCT\.md$
13
+^\.bash_profile$

+ 6
- 5
DESCRIPTION View File

@@ -1,4 +1,4 @@
1
-Package: metis.lite
1
+Package: metis
2 2
 Type: Package
3 3
 Title: Access and Query Amazon Athena via DBI/JDBC
4 4
 Version: 0.3.0
@@ -9,9 +9,9 @@ Authors@R: c(
9 9
   )
10 10
 Maintainer: Bob Rudis <bob@rud.is>
11 11
 Encoding: UTF-8
12
-Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
13
-    perform queries and retrieve query results. A lightweight 'RJDBC' implementation 
14
-    is included along with additional helpers for 'dplyr'/'dplyr' suppprt.
12
+Description: Methods are provided to connect to 'Amazon' 'Athena', lookup 
13
+    schemas/tables, perform queries and retrieve query results using the
14
+    Athena JDBC driver found in 'metis.jars'.
15 15
 SystemRequirements: JDK 1.8+
16 16
 License: MIT + file LICENSE
17 17
 Suggests:
@@ -19,11 +19,12 @@ Suggests:
19 19
     covr
20 20
 Depends:
21 21
     R (>= 3.2.0),
22
+    metis.jars,
22 23
     RJDBC
23 24
 Imports:
24 25
     rJava,
25 26
     DBI,
26 27
     bit64,
27
-    dbplyr,
28
+    methods,
28 29
     aws.signature
29 30
 RoxygenNote: 6.1.1

+ 3
- 3
NAMESPACE View File

@@ -2,9 +2,7 @@
2 2
 
3 3
 export(Athena)
4 4
 export(athena_connect)
5
-export(db_data_type.AthenaConnection)
6 5
 export(read_credentials)
7
-export(sql_translate_env.AthenaConnection)
8 6
 export(use_credentials)
9 7
 exportClasses(AthenaConnection)
10 8
 exportClasses(AthenaDriver)
@@ -21,7 +19,9 @@ exportMethods(fetch)
21 19
 import(DBI)
22 20
 import(RJDBC)
23 21
 import(bit64)
24
-import(dbplyr)
22
+import(metis.jars)
25 23
 importFrom(aws.signature,read_credentials)
26 24
 importFrom(aws.signature,use_credentials)
25
+importFrom(methods,as)
26
+importFrom(methods,callNextMethod)
27 27
 importFrom(rJava,.jcall)

+ 7
- 1
R/jdbc.r View File

@@ -21,12 +21,13 @@ setClass(
21 21
 
22 22
 #' AthenaJDBC
23 23
 #'
24
+#' @param identifier.quote how to quote identifiers
24 25
 #' @export
25 26
 Athena <- function(identifier.quote = '`') {
26 27
 
27 28
   JDBC(
28 29
     driverClass = "com.simba.athena.jdbc.Driver",
29
-    system.file("java", "AthenaJDBC42_2.0.6.jar", package = "metis.lite"),
30
+    metis.jars::metis_jar_path(),
30 31
     identifier.quote = identifier.quote
31 32
   ) -> drv
32 33
 
@@ -44,6 +45,7 @@ Athena <- function(identifier.quote = '`') {
44 45
 #' - `ComplexTypeColumnLength`: <int> The maximum data length for `ARRAY`, `MAP`, and `STRUCT` columns. Default `65535L`
45 46
 #' - `StringColumnLength`: <int> The maximum data length for `STRING` columns. Default `255L`
46 47
 #'
48
+#' @param drv driver
47 49
 #' @param provider JDBC auth provider (ideally leave default)
48 50
 #' @param region AWS region the Athena tables are in
49 51
 #' @param s3_staging_dir A write-able bucket on S3 that you have permissions for
@@ -54,6 +56,7 @@ Athena <- function(identifier.quote = '`') {
54 56
 #'     of data in logs. Set this to a temporary directory or something log4j can use. For
55 57
 #'     `log_level` use the names ("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE") or
56 58
 #'     their corresponding integer values 0-6.
59
+#' @param fetch_size Athena results fetch size
57 60
 #' @param ... passed on to the driver. See Details.
58 61
 #' @references [Connect with JDBC](https://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html);
59 62
 #'     [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf)
@@ -109,6 +112,9 @@ setMethod(
109 112
 
110 113
 #' AthenaJDBC
111 114
 #'
115
+#' @param jc job ref
116
+#' @param identifier.quote how to quote identifiers
117
+#' @param fetch_size Athena results fetch size
112 118
 #' @export
113 119
 setClass("AthenaConnection", representation("JDBCConnection", jc="jobjRef", identifier.quote="character", fetch_size="integer"))
114 120
 

+ 0
- 48
R/metis-lite-package.R View File

@@ -1,48 +0,0 @@
1
-#' Access and Query Amazon Athena via DBI/JDBC
2
-#'
3
-#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
4
-#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation
5
-#' is included along with an interface to the 'AWS' command-line utility.
6
-#'
7
-#' @name metis.lite
8
-#'
9
-#' @section IMPORTANT:
10
-#'
11
-#' Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
12
-#' Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid
13
-#' signals being passed on to the JVM owner. That has to be done _before_ `rJava` is
14
-#' loaded so you either need to remember to put it at the top of all scripts _or_ stick this
15
-#' in your local #' `~/.Rprofile` and/or sitewide `Rprofile`:
16
-#'
17
-#'
18
-#'     if (!grepl("-Xrs", getOption("java.parameters", ""))) {
19
-#'       options(
20
-#'         "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
21
-#'       )
22
-#'     }
23
-#'
24
-#' @md
25
-#' @encoding UTF-8
26
-#' @keywords internal
27
-#' @docType package
28
-#' @author Bob Rudis (bob@@rud.is)
29
-#' @import RJDBC DBI bit64 dbplyr
30
-#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf)
31
-NULL
32
-
33
-
34
-#' Use Credentials from .aws/credentials File
35
-#'
36
-#' @md
37
-#' @importFrom aws.signature use_credentials read_credentials
38
-#' @references [aws.signature::use_credentials()] /  [aws.signature::read_credentials()]
39
-#' @name use_credentials
40
-#' @rdname use_credentials
41
-#' @inheritParams aws.signature::use_credentials
42
-#' @export
43
-NULL
44
-
45
-#' @name read_credentials
46
-#' @rdname use_credentials
47
-#' @export
48
-NULL

+ 33
- 0
R/metis-package.R View File

@@ -0,0 +1,33 @@
1
+#' Access and Query Amazon Athena via DBI/JDBC
2
+#'
3
+#' Methods are provided to connect to 'Amazon' 'Athena', lookup
4
+#' schemas/tables, perform queries and retrieve query results using the
5
+#' Athena JDBC driver found in 'metis.jars'.
6
+#'
7
+#' @name metis
8
+#'
9
+#' @md
10
+#' @encoding UTF-8
11
+#' @keywords internal
12
+#' @docType package
13
+#' @author Bob Rudis (bob@@rud.is)
14
+#' @import RJDBC DBI bit64 metis.jars
15
+#' @importFrom methods as callNextMethod
16
+#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf)
17
+NULL
18
+
19
+#' Use Credentials from .aws/credentials File
20
+#'
21
+#' @md
22
+#' @importFrom aws.signature use_credentials read_credentials
23
+#' @references [aws.signature::use_credentials()] /  [aws.signature::read_credentials()]
24
+#' @name use_credentials
25
+#' @rdname use_credentials
26
+#' @inheritParams aws.signature::use_credentials
27
+#' @export
28
+NULL
29
+
30
+#' @name read_credentials
31
+#' @rdname use_credentials
32
+#' @export
33
+NULL

+ 0
- 124
R/sql_translate_env.R View File

@@ -1,124 +0,0 @@
1
-#' @rdname Athena
2
-#' @keywords internal
3
-#' @export
4
-db_data_type.AthenaConnection <- function(con, fields, ...) {
5
-  print("\n\n\ndb_data_type\n\n\n")
6
-  data_type <- function(x) {
7
-    switch(
8
-      class(x)[1],
9
-      integer64 = "BIGINT",
10
-      logical = "BOOLEAN",
11
-      integer = "INTEGER",
12
-      numeric = "DOUBLE",
13
-      factor =  "CHARACTER",
14
-      character = "CHARACTER",
15
-      Date = "DATE",
16
-      POSIXct = "TIMESTAMP",
17
-      stop("Can't map type ", paste(class(x), collapse = "/"),
18
-           " to a supported database type.")
19
-    )
20
-  }
21
-  vapply(fields, data_type, character(1))
22
-}
23
-
24
-#' @rdname Athena
25
-#' @keywords internal
26
-#' @export
27
-sql_translate_env.AthenaConnection <- function(con) {
28
-
29
-  x <- con
30
-
31
-  dbplyr::sql_variant(
32
-
33
-    scalar = dbplyr::sql_translator(
34
-      .parent = dbplyr::base_scalar,
35
-      `!=` = dbplyr::sql_infix("<>"),
36
-      as.integer64 = function(x) dbplyr::build_sql("CAST(", x, "AS BIGINT)"),
37
-      as.numeric = function(x) dbplyr::build_sql("CAST(", x, " AS DOUBLE)"),
38
-      as.character = function(x) dbplyr::build_sql("CAST(", x, " AS CHARACTER)"),
39
-      as.date = function(x) dbplyr::build_sql("CAST(", x, " AS DATE)"),
40
-      as.Date = function(x) dbplyr::build_sql("CAST(", x, " AS DATE)"),
41
-      as.POSIXct = function(x) dbplyr::build_sql("CAST(", x, " AS TIMESTAMP)"),
42
-      as.posixct = function(x) dbplyr::build_sql("CAST(", x, " AS TIMESTAMP)"),
43
-      as.logical = function(x) dbplyr::build_sql("CAST(", x, " AS BOOLEAN)"),
44
-      date_part = function(x, y) dbplyr::build_sql("DATE_PART(", x, ",", y ,")"),
45
-      grepl = function(x, y) dbplyr::build_sql("CONTAINS(", y, ", ", x, ")"),
46
-      gsub = function(x, y, z) dbplyr::build_sql("REGEXP_REPLACE(", z, ", ", x, ",", y ,")"),
47
-      trimws = function(x) dbplyr::build_sql("TRIM(both ' ' FROM ", x, ")"),
48
-      cbrt = dbplyr::sql_prefix("CBRT", 1),
49
-      degrees = dbplyr::sql_prefix("DEGREES", 1),
50
-      e = dbplyr::sql_prefix("E", 0),
51
-      row_number = dbplyr::sql_prefix("row_number", 0),
52
-      lshift = dbplyr::sql_prefix("LSHIFT", 2),
53
-      mod = dbplyr::sql_prefix("MOD", 2),
54
-      age = dbplyr::sql_prefix("AGE", 1),
55
-      negative = dbplyr::sql_prefix("NEGATIVE", 1),
56
-      pi = dbplyr::sql_prefix("PI", 0),
57
-      pow = dbplyr::sql_prefix("POW", 2),
58
-      radians = dbplyr::sql_prefix("RADIANS", 1),
59
-      rand = dbplyr::sql_prefix("RAND", 0),
60
-      rshift = dbplyr::sql_prefix("RSHIFT", 2),
61
-      trunc = dbplyr::sql_prefix("TRUNC", 2),
62
-      contains = dbplyr::sql_prefix("CONTAINS", 2),
63
-      convert_to = dbplyr::sql_prefix("CONVERT_TO", 2),
64
-      convert_from = dbplyr::sql_prefix("CONVERT_FROM", 2),
65
-      string_binary = dbplyr::sql_prefix("STRING_BINARY", 1),
66
-      binary_string = dbplyr::sql_prefix("BINARY_STRING", 1),
67
-      to_char = dbplyr::sql_prefix("TO_CHAR", 2),
68
-      to_date = dbplyr::sql_prefix("TO_DATE", 2),
69
-      to_number = dbplyr::sql_prefix("TO_NUMBER", 2),
70
-      char_to_timestamp = dbplyr::sql_prefix("TO_TIMESTAMP", 2),
71
-      double_to_timestamp = dbplyr::sql_prefix("TO_TIMESTAMP", 1),
72
-      char_length = dbplyr::sql_prefix("CHAR_LENGTH", 1),
73
-      flatten = dbplyr::sql_prefix("FLATTEN", 1),
74
-      kvgen = dbplyr::sql_prefix("KVGEN", 1),
75
-      repeated_count = dbplyr::sql_prefix("REPEATED_COUNT", 1),
76
-      repeated_contains = dbplyr::sql_prefix("REPEATED_CONTAINS", 2),
77
-      ilike = dbplyr::sql_prefix("ILIKE", 2),
78
-      init_cap = dbplyr::sql_prefix("INIT_CAP", 1),
79
-      length = dbplyr::sql_prefix("LENGTH", 1),
80
-      lower = dbplyr::sql_prefix("LOWER", 1),
81
-      tolower = dbplyr::sql_prefix("LOWER", 1),
82
-      ltrim = dbplyr::sql_prefix("LTRIM", 2),
83
-      nullif = dbplyr::sql_prefix("NULLIF", 2),
84
-      position = function(x, y) dbplyr::build_sql("POSITION(", x, " IN ", y, ")"),
85
-      regexp_replace = dbplyr::sql_prefix("REGEXP_REPLACE", 3),
86
-      rtrim = dbplyr::sql_prefix("RTRIM", 2),
87
-      rpad = dbplyr::sql_prefix("RPAD", 2),
88
-      rpad_with = dbplyr::sql_prefix("RPAD", 3),
89
-      lpad = dbplyr::sql_prefix("LPAD", 2),
90
-      lpad_with = dbplyr::sql_prefix("LPAD", 3),
91
-      strpos = dbplyr::sql_prefix("STRPOS", 2),
92
-      substr = dbplyr::sql_prefix("SUBSTR", 3),
93
-      trim = function(x, y, z) dbplyr::build_sql("TRIM(", x, " ", y, " FROM ", z, ")"),
94
-      upper = dbplyr::sql_prefix("UPPER", 1),
95
-      toupper = dbplyr::sql_prefix("UPPER", 1)
96
-    ),
97
-
98
-    aggregate = dbplyr::sql_translator(
99
-      .parent = dbplyr::base_agg,
100
-      n = function() dbplyr::sql("COUNT(*)"),
101
-      cor = dbplyr::sql_prefix("CORR"),
102
-      cov = dbplyr::sql_prefix("COVAR_SAMP"),
103
-      sd =  dbplyr::sql_prefix("STDDEV_SAMP"),
104
-      var = dbplyr::sql_prefix("VAR_SAMP"),
105
-      n_distinct = function(x) {
106
-        dbplyr::build_sql(dbplyr::sql("COUNT(DISTINCT "), x, dbplyr::sql(")"))
107
-      }
108
-    ),
109
-
110
-    window = dbplyr::sql_translator(
111
-      .parent = dbplyr::base_win,
112
-      n = function() { dbplyr::win_over(dbplyr::sql("count(*)"),
113
-                                        partition = dbplyr::win_current_group()) },
114
-      cor = dbplyr::win_recycled("corr"),
115
-      cov = dbplyr::win_recycled("covar_samp"),
116
-      sd =  dbplyr::win_recycled("stddev_samp"),
117
-      var = dbplyr::win_recycled("var_samp"),
118
-      all = dbplyr::win_recycled("bool_and"),
119
-      any = dbplyr::win_recycled("bool_or")
120
-    )
121
-
122
-  )
123
-
124
-}

+ 12
- 4
R/z-dbGetQuery.R View File

@@ -24,27 +24,34 @@ list(
24 24
   "1111" = as.character # OTHER
25 25
 ) -> .jdbc_converters
26 26
 
27
+#' Retrieve connection/driver/database metadata
28
+#'
29
+#' @param dbObj driver/connection
30
+#' @param ... unused
27 31
 #' @export
28 32
 #' @keywords internal
29 33
 setMethod("dbGetInfo", "AthenaDriver", def=function(dbObj, ...)
30 34
   list(
31 35
     name = "AthenaJDBC",
32
-    driver_version = list.files(system.file("java", package="metis.lite"), "jar$")[1],
33
-    package_version = utils::packageVersion("metis.lite")
36
+    driver_version = metis.jars::simba_driver_version(),
37
+    package_version = utils::packageVersion("metis.jars")
34 38
   )
35 39
 )
36 40
 
41
+#' Retrieve connection/driver//database metadata
42
+#'
43
+#' @param dbObj driver/connection
44
+#' @param ... unused
37 45
 #' @export
38 46
 #' @keywords internal
39 47
 setMethod("dbGetInfo", "AthenaConnection", def=function(dbObj, ...)
40 48
   list(
41 49
     name = "AthenaJDBC",
42 50
     driver_version = list.files(system.file("java", package="metis.lite"), "jar$")[1],
43
-    package_version = utils::packageVersion("metis.lite")
51
+    package_version = utils::packageVersion("metis")
44 52
   )
45 53
 )
46 54
 
47
-
48 55
 #' Fetch records from a previously executed query
49 56
 #'
50 57
 #' Fetch the next `n` elements (rows) from the result set and return them
@@ -56,6 +63,7 @@ setMethod("dbGetInfo", "AthenaConnection", def=function(dbObj, ...)
56 63
 #'   or `n = Inf`
57 64
 #'   to retrieve all pending records.  Some implementations may recognize other
58 65
 #'   special values.
66
+#' @param block clock size
59 67
 #' @param ... Other arguments passed on to methods.
60 68
 #' @export
61 69
 setMethod(

+ 0
- 9
R/zzz.R View File

@@ -1,11 +1,2 @@
1 1
 .onLoad <- function(libname, pkgname) {
2
-  rJava::.jpackage(pkgname, jars = "*", lib.loc = libname)
3
-  rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE))
4
-  o <- getOption("java.parameters", "")
5
-  if (!any(grepl("-Xrs", o))) {
6
-    packageStartupMessage(
7
-      "Did not find '-Xrs' in java.parameters option. Until rJava is updated, ",
8
-      "please set this up in your/an Rprofile or at the start of scripts."
9
-    )
10
-  }
11 2
 }

+ 15
- 46
README.Rmd View File

@@ -10,20 +10,11 @@ Access and Query Amazon Athena via DBI/JDBC
10 10
 
11 11
 ## Description
12 12
 
13
-In Greek mythology, Metis was Athena's "helper" so methods are provided to help you accessing and querying Amazon Athena via DBI/JDBC and/or `dplyr`.
14
-#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
13
+In Greek mythology, Metis was Athena's "helper" so... 
15 14
 
16
-## IMPORTANT
15
+Methods are provided to connect to 'Amazon' 'Athena', lookup schemas/tables, 
16
+perform queries and retrieve query results via the included JDBC DBI driver.
17 17
 
18
-Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting Athena JDBC calls crash the R interpreter. You need to set the `-Xrs` option to avoid signals being passed on to the JVM owner. That has to be done _before_ `rJava` is loaded so you either need to remember to put it at the top of all scripts _or_ stick this in your local `~/.Rprofile` and/or sitewide `Rprofile`:
19
-
20
-```r
21
-if (!grepl("-Xrs", getOption("java.parameters", ""))) {
22
-  options(
23
-    "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
24
-  )
25
-}
26
-```
27 18
 ## What's Inside The Tin?
28 19
 
29 20
 The following functions are implemented:
@@ -57,11 +48,11 @@ Pulled in from other `cloudyr` pkgs:
57 48
 ## Installation
58 49
 
59 50
 ```{r eval=FALSE}
60
-devtools::install_git("https://git.sr.ht/~hrbrmstr/metis-lite")
51
+devtools::install_git("https://git.sr.ht/~hrbrmstr/metis")
61 52
 # OR
62
-devtools::install_gitlab("hrbrmstr/metis-lite")
53
+devtools::install_gitlab("hrbrmstr/metis")
63 54
 # OR
64
-devtools::install_github("hrbrmstr/metis-lite")
55
+devtools::install_github("hrbrmstr/metis")
65 56
 ```
66 57
 
67 58
 ```{r message=FALSE, warning=FALSE, include=FALSE}
@@ -71,22 +62,20 @@ options(width=120)
71 62
 ## Usage
72 63
 
73 64
 ```{r message=FALSE, warning=FALSE}
74
-library(metis.lite)
65
+library(metis)
75 66
 
76 67
 # current verison
77
-packageVersion("metis.lite")
68
+packageVersion("metis")
78 69
 ```
79 70
 
80 71
 ```{r message=FALSE, warning=FALSE}
81 72
 library(rJava)
82 73
 library(RJDBC)
83
-library(metis.lite)
84
-library(magrittr)
85
-library(dbplyr)
86
-library(dplyr)
74
+library(metis)
75
+library(magrittr) 
87 76
 
88 77
 dbConnect(
89
-  drv = metis.lite::Athena(),
78
+  drv = metis::Athena(),
90 79
   schema_name = "sampledb",
91 80
   provider = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
92 81
   AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
@@ -100,7 +89,7 @@ dbExistsTable(con, "elb_logs", schema="sampledb")
100 89
 dbListFields(con, "elb_logs", "sampledb")
101 90
 
102 91
 dbGetQuery(con, "SELECT * FROM sampledb.elb_logs LIMIT 10") %>% 
103
-  glimpse()
92
+  dplyr::glimpse()
104 93
 ```
105 94
 
106 95
 ### Check types
@@ -124,31 +113,11 @@ SELECT
124 113
 FROM elb_logs
125 114
 LIMIT 1
126 115
 ") %>% 
127
-  glimpse()
116
+  dplyr::glimpse()
128 117
 ```
129 118
 
130
-#### dplyr
131
-
132
-```{r}
133
-tbl(con, sql("
134
-SELECT
135
-  CAST('chr' AS CHAR(4)) achar,
136
-  CAST('varchr' AS VARCHAR) avarchr,
137
-  CAST(SUBSTR(timestamp, 1, 10) AS DATE) AS tsday,
138
-  CAST(100.1 AS DOUBLE) AS justadbl,
139
-  CAST(127 AS TINYINT) AS asmallint,
140
-  CAST(100 AS INTEGER) AS justanint,
141
-  CAST(100000000000000000 AS BIGINT) AS abigint,
142
-  CAST(('GET' = 'GET') AS BOOLEAN) AS is_get,
143
-  ARRAY[1, 2, 3] AS arr,
144
-  ARRAY['1', '2, 3', '4'] AS arr,
145
-  MAP(ARRAY['foo', 'bar'], ARRAY[1, 2]) AS mp,
146
-  CAST(ROW(1, 2.0) AS ROW(x BIGINT, y DOUBLE)) AS rw,
147
-  CAST('{\"a\":1}' AS JSON) js
148
-FROM elb_logs
149
-LIMIT 1
150
-")) %>% 
151
-  glimpse()
119
+```{r cloc}
120
+cloc::cloc_pkg_md()
152 121
 ```
153 122
 
154 123
 ## Code of Conduct

+ 30
- 80
README.md View File

@@ -5,27 +5,11 @@ Access and Query Amazon Athena via DBI/JDBC
5 5
 
6 6
 ## Description
7 7
 
8
-In Greek mythology, Metis was Athena’s “helper” so methods are provided
9
-to help you accessing and querying Amazon Athena via DBI/JDBC and/or
10
-`dplyr`. \#’ Methods are provides to connect to ‘Amazon’ ‘Athena’,
11
-lookup schemas/tables,
8
+In Greek mythology, Metis was Athena’s “helper” so…
12 9
 
13
-## IMPORTANT
14
-
15
-Since R 3.5 (I don’t remember this happening in R 3.4.x) signals sent
16
-from interrupting Athena JDBC calls crash the R interpreter. You need to
17
-set the `-Xrs` option to avoid signals being passed on to the JVM owner.
18
-That has to be done *before* `rJava` is loaded so you either need to
19
-remember to put it at the top of all scripts *or* stick this in your
20
-local `~/.Rprofile` and/or sitewide `Rprofile`:
21
-
22
-``` r
23
-if (!grepl("-Xrs", getOption("java.parameters", ""))) {
24
-  options(
25
-    "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
26
-  )
27
-}
28
-```
10
+Methods are provided to connect to ‘Amazon’ ‘Athena’, lookup
11
+schemas/tables, perform queries and retrieve query results via the
12
+included JDBC DBI driver.
29 13
 
30 14
 ## What’s Inside The Tin?
31 15
 
@@ -60,20 +44,20 @@ Pulled in from other `cloudyr` pkgs:
60 44
 ## Installation
61 45
 
62 46
 ``` r
63
-devtools::install_git("https://git.sr.ht/~hrbrmstr/metis-lite")
47
+devtools::install_git("https://git.sr.ht/~hrbrmstr/metis")
64 48
 # OR
65
-devtools::install_gitlab("hrbrmstr/metis-lite")
49
+devtools::install_gitlab("hrbrmstr/metis")
66 50
 # OR
67
-devtools::install_github("hrbrmstr/metis-lite")
51
+devtools::install_github("hrbrmstr/metis")
68 52
 ```
69 53
 
70 54
 ## Usage
71 55
 
72 56
 ``` r
73
-library(metis.lite)
57
+library(metis)
74 58
 
75 59
 # current verison
76
-packageVersion("metis.lite")
60
+packageVersion("metis")
77 61
 ```
78 62
 
79 63
     ## [1] '0.3.0'
@@ -81,10 +65,8 @@ packageVersion("metis.lite")
81 65
 ``` r
82 66
 library(rJava)
83 67
 library(RJDBC)
84
-library(metis.lite)
85
-library(magrittr)
86
-library(dbplyr)
87
-library(dplyr)
68
+library(metis)
69
+library(magrittr) 
88 70
 
89 71
 dbConnect(
90 72
   drv = metis.lite::Athena(),
@@ -116,26 +98,26 @@ dbListFields(con, "elb_logs", "sampledb")
116 98
 
117 99
 ``` r
118 100
 dbGetQuery(con, "SELECT * FROM sampledb.elb_logs LIMIT 10") %>% 
119
-  glimpse()
101
+  dplyr::glimpse()
120 102
 ```
121 103
 
122 104
     ## Observations: 10
123 105
     ## Variables: 16
124
-    ## $ timestamp             <chr> "2014-09-29T18:18:51.826955Z", "2014-09-29T18:18:51.920462Z", "2014-09-29T18:18:52.2725
106
+    ## $ timestamp             <chr> "2014-09-29T03:24:38.169500Z", "2014-09-29T03:25:09.029469Z", "2014-09-29T03:25:39.8676
125 107
     ## $ elbname               <chr> "lb-demo", "lb-demo", "lb-demo", "lb-demo", "lb-demo", "lb-demo", "lb-demo", "lb-demo",…
126
-    ## $ requestip             <chr> "255.48.150.122", "249.213.227.93", "245.108.120.229", "241.112.203.216", "241.43.107.2
127
-    ## $ requestport           <int> 62096, 62096, 62096, 62096, 56454, 33254, 18918, 64352, 1651, 56454
128
-    ## $ backendip             <chr> "244.238.214.120", "248.99.214.228", "243.3.190.175", "246.235.181.255", "241.112.203.2
129
-    ## $ backendport           <int> 8888, 8888, 8888, 8888, 8888, 8888, 8888, 8888, 8888, 8888
130
-    ## $ requestprocessingtime <dbl> 9.0e-05, 9.7e-05, 8.7e-05, 9.4e-05, 7.6e-05, 8.3e-05, 6.3e-05, 5.4e-05, 8.2e-05, 8.7e-05
131
-    ## $ backendprocessingtime <dbl> 0.007410, 0.256533, 0.442659, 0.016772, 0.035036, 0.029892, 0.034148, 0.014858, 0.01518
132
-    ## $ clientresponsetime    <dbl> 0.000055, 0.000075, 0.000131, 0.000078, 0.000057, 0.000043, 0.000033, 0.000043, 0.00007…
133
-    ## $ elbresponsecode       <chr> "302", "302", "200", "200", "200", "200", "200", "200", "200", "200"
134
-    ## $ backendresponsecode   <chr> "200", "200", "200", "200", "200", "200", "200", "200", "200", "200"
108
+    ## $ requestip             <chr> "253.89.30.138", "248.64.121.231", "245.21.209.210", "244.77.57.59", "244.185.170.87", 
109
+    ## $ requestport           <int> 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159
110
+    ## $ backendip             <chr> "253.89.30.138", "244.77.57.59", "240.105.192.251", "253.89.30.138", "248.64.121.231", 
111
+    ## $ backendport           <int> 8888, 8888, 8888, 8899, 8888, 8888, 8888, 8888, 8888, 8888
112
+    ## $ requestprocessingtime <dbl> 7.5e-05, 9.1e-05, 9.0e-05, 9.5e-05, 8.9e-05, 9.3e-05, 8.7e-05, 9.2e-05, 9.0e-05, 9.1e-05
113
+    ## $ backendprocessingtime <dbl> 0.047465, 0.044693, 0.045687, 0.051089, 0.045445, 0.045845, 0.046027, 0.045039, 0.05010
114
+    ## $ clientresponsetime    <dbl> 6.5e-05, 7.2e-05, 6.4e-05, 7.0e-05, 5.4e-05, 6.7e-05, 5.7e-05, 4.6e-05, 8.7e-05, 4.9e-05
115
+    ## $ elbresponsecode       <chr> "200", "200", "200", "200", "200", "200", "200", "200", "200", "200"
116
+    ## $ backendresponsecode   <chr> "200", "200", "400", "200", "404", "200", "403", "404", "200", "200"
135 117
     ## $ receivedbytes         <S3: integer64> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
136
-    ## $ sentbytes             <S3: integer64> 0, 0, 58402, 152213, 20766, 32370, 3408, 3884, 84245, 3831
118
+    ## $ sentbytes             <S3: integer64> 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
137 119
     ## $ requestverb           <chr> "GET", "GET", "GET", "GET", "GET", "GET", "GET", "GET", "GET", "GET"
138
-    ## $ url                   <chr> "http://www.abcxyz.com:80/", "http://www.abcxyz.com:80/accounts/login/?next=/", "http:/…
120
+    ## $ url                   <chr> "http://www.abcxyz.com:80/jobbrowser/?format=json&state=running&user=248nnm5", "http://…
139 121
     ## $ protocol              <chr> "HTTP/1.1", "HTTP/1.1", "HTTP/1.1", "HTTP/1.1", "HTTP/1.1", "HTTP/1.1", "HTTP/1.1", "HT…
140 122
 
141 123
 ### Check types
@@ -159,7 +141,7 @@ SELECT
159 141
 FROM elb_logs
160 142
 LIMIT 1
161 143
 ") %>% 
162
-  glimpse()
144
+  dplyr::glimpse()
163 145
 ```
164 146
 
165 147
     ## Observations: 1
@@ -178,46 +160,14 @@ LIMIT 1
178 160
     ## $ rw        <chr> "{x=1, y=2.0}"
179 161
     ## $ js        <chr> "\"{\\\"a\\\":1}\""
180 162
 
181
-#### dplyr
182
-
183 163
 ``` r
184
-tbl(con, sql("
185
-SELECT
186
-  CAST('chr' AS CHAR(4)) achar,
187
-  CAST('varchr' AS VARCHAR) avarchr,
188
-  CAST(SUBSTR(timestamp, 1, 10) AS DATE) AS tsday,
189
-  CAST(100.1 AS DOUBLE) AS justadbl,
190
-  CAST(127 AS TINYINT) AS asmallint,
191
-  CAST(100 AS INTEGER) AS justanint,
192
-  CAST(100000000000000000 AS BIGINT) AS abigint,
193
-  CAST(('GET' = 'GET') AS BOOLEAN) AS is_get,
194
-  ARRAY[1, 2, 3] AS arr,
195
-  ARRAY['1', '2, 3', '4'] AS arr,
196
-  MAP(ARRAY['foo', 'bar'], ARRAY[1, 2]) AS mp,
197
-  CAST(ROW(1, 2.0) AS ROW(x BIGINT, y DOUBLE)) AS rw,
198
-  CAST('{\"a\":1}' AS JSON) js
199
-FROM elb_logs
200
-LIMIT 1
201
-")) %>% 
202
-  glimpse()
164
+cloc::cloc_pkg_md()
203 165
 ```
204 166
 
205
-    ## Observations: ??
206
-    ## Variables: 13
207
-    ## Database: AthenaConnection
208
-    ## $ achar     <chr> "chr "
209
-    ## $ avarchr   <chr> "varchr"
210
-    ## $ tsday     <date> 2014-09-27
211
-    ## $ justadbl  <dbl> 100.1
212
-    ## $ asmallint <int> 127
213
-    ## $ justanint <int> 100
214
-    ## $ abigint   <S3: integer64> 100000000000000000
215
-    ## $ is_get    <lgl> TRUE
216
-    ## $ arr       <chr> "1, 2, 3"
217
-    ## $ arr       <chr> "1, 2, 3, 4"
218
-    ## $ mp        <chr> "{bar=2, foo=1}"
219
-    ## $ rw        <chr> "{x=1, y=2.0}"
220
-    ## $ js        <chr> "\"{\\\"a\\\":1}\""
167
+| Lang | \# Files |  (%) | LoC |  (%) | Blank lines |  (%) | \# Lines |  (%) |
168
+| :--- | -------: | ---: | --: | ---: | ----------: | ---: | -------: | ---: |
169
+| R    |        8 | 0.89 | 232 | 0.85 |          77 | 0.71 |      160 | 0.76 |
170
+| Rmd  |        1 | 0.11 |  42 | 0.15 |          32 | 0.29 |       51 | 0.24 |
221 171
 
222 172
 ## Code of Conduct
223 173
 

BIN
inst/java/AthenaJDBC42_2.0.6.jar View File


+ 0
- 1
inst/java/log4j.properties View File

@@ -1 +0,0 @@
1
-log4j.rootLogger=WARN

+ 4
- 8
man/Athena.Rd View File

@@ -1,18 +1,14 @@
1 1
 % Generated by roxygen2: do not edit by hand
2
-% Please edit documentation in R/jdbc.r, R/sql_translate_env.R
2
+% Please edit documentation in R/jdbc.r
3 3
 \name{Athena}
4 4
 \alias{Athena}
5
-\alias{db_data_type.AthenaConnection}
6
-\alias{sql_translate_env.AthenaConnection}
7 5
 \title{AthenaJDBC}
8 6
 \usage{
9 7
 Athena(identifier.quote = "`")
10
-
11
-db_data_type.AthenaConnection(con, fields, ...)
12
-
13
-sql_translate_env.AthenaConnection(con)
8
+}
9
+\arguments{
10
+\item{identifier.quote}{how to quote identifiers}
14 11
 }
15 12
 \description{
16 13
 AthenaJDBC
17 14
 }
18
-\keyword{internal}

+ 7
- 0
man/AthenaConnection-class.Rd View File

@@ -4,6 +4,13 @@
4 4
 \name{AthenaConnection-class}
5 5
 \alias{AthenaConnection-class}
6 6
 \title{AthenaJDBC}
7
+\arguments{
8
+\item{jc}{job ref}
9
+
10
+\item{identifier.quote}{how to quote identifiers}
11
+
12
+\item{fetch_size}{Athena results fetch size}
13
+}
7 14
 \description{
8 15
 AthenaJDBC
9 16
 }

+ 4
- 0
man/dbConnect-AthenaDriver-method.Rd View File

@@ -14,6 +14,8 @@
14 14
   socket_timeout = 10000, log_path = "", log_level = 0, ...)
15 15
 }
16 16
 \arguments{
17
+\item{drv}{driver}
18
+
17 19
 \item{provider}{JDBC auth provider (ideally leave default)}
18 20
 
19 21
 \item{region}{AWS region the Athena tables are in}
@@ -22,6 +24,8 @@
22 24
 
23 25
 \item{schema_name}{LOL if only this actually worked with Amazon's hacked Presto driver}
24 26
 
27
+\item{fetch_size}{Athena results fetch size}
28
+
25 29
 \item{max_error_retries, connection_timeout, socket_timeout}{technical connection info that you should only muck with if you know what you're doing.}
26 30
 
27 31
 \item{log_path, log_level}{The Athena JDBC driver can (shockingly) provide a decent bit

+ 18
- 0
man/dbGetInfo-AthenaConnection-method.Rd View File

@@ -0,0 +1,18 @@
1
+% Generated by roxygen2: do not edit by hand
2
+% Please edit documentation in R/z-dbGetQuery.R
3
+\docType{methods}
4
+\name{dbGetInfo,AthenaConnection-method}
5
+\alias{dbGetInfo,AthenaConnection-method}
6
+\title{Retrieve connection/driver//database metadata}
7
+\usage{
8
+\S4method{dbGetInfo}{AthenaConnection}(dbObj, ...)
9
+}
10
+\arguments{
11
+\item{dbObj}{driver/connection}
12
+
13
+\item{...}{unused}
14
+}
15
+\description{
16
+Retrieve connection/driver//database metadata
17
+}
18
+\keyword{internal}

+ 18
- 0
man/dbGetInfo-AthenaDriver-method.Rd View File

@@ -0,0 +1,18 @@
1
+% Generated by roxygen2: do not edit by hand
2
+% Please edit documentation in R/z-dbGetQuery.R
3
+\docType{methods}
4
+\name{dbGetInfo,AthenaDriver-method}
5
+\alias{dbGetInfo,AthenaDriver-method}
6
+\title{Retrieve connection/driver/database metadata}
7
+\usage{
8
+\S4method{dbGetInfo}{AthenaDriver}(dbObj, ...)
9
+}
10
+\arguments{
11
+\item{dbObj}{driver/connection}
12
+
13
+\item{...}{unused}
14
+}
15
+\description{
16
+Retrieve connection/driver/database metadata
17
+}
18
+\keyword{internal}

+ 2
- 0
man/fetch-AthenaResult-numeric-method.Rd View File

@@ -16,6 +16,8 @@ or `n = Inf`
16 16
 to retrieve all pending records.  Some implementations may recognize other
17 17
 special values.}
18 18
 
19
+\item{block}{clock size}
20
+
19 21
 \item{...}{Other arguments passed on to methods.}
20 22
 }
21 23
 \description{

+ 20
- 0
man/metis.Rd View File

@@ -0,0 +1,20 @@
1
+% Generated by roxygen2: do not edit by hand
2
+% Please edit documentation in R/metis-package.R
3
+\docType{package}
4
+\encoding{UTF-8}
5
+\name{metis}
6
+\alias{metis}
7
+\alias{metis-package}
8
+\title{Access and Query Amazon Athena via DBI/JDBC}
9
+\description{
10
+Methods are provided to connect to 'Amazon' 'Athena', lookup
11
+schemas/tables, perform queries and retrieve query results using the
12
+Athena JDBC driver found in 'metis.jars'.
13
+}
14
+\references{
15
+\href{https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf}{Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide}
16
+}
17
+\author{
18
+Bob Rudis (bob@rud.is)
19
+}
20
+\keyword{internal}

+ 0
- 35
man/metis.lite.Rd View File

@@ -1,35 +0,0 @@
1
-% Generated by roxygen2: do not edit by hand
2
-% Please edit documentation in R/metis-lite-package.R
3
-\docType{package}
4
-\encoding{UTF-8}
5
-\name{metis.lite}
6
-\alias{metis.lite}
7
-\alias{metis.lite-package}
8
-\title{Access and Query Amazon Athena via DBI/JDBC}
9
-\description{
10
-Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
11
-perform queries and retrieve query results. A lightweight 'RJDBC' implementation
12
-is included along with an interface to the 'AWS' command-line utility.
13
-}
14
-\section{IMPORTANT}{
15
-
16
-
17
-Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
18
-Athena JDBC calls crash the R #' interpreter. You need to set the \code{-Xrs} option to avoid
19
-signals being passed on to the JVM owner. That has to be done \emph{before} \code{rJava} is
20
-loaded so you either need to remember to put it at the top of all scripts \emph{or} stick this
21
-in your local #' \code{~/.Rprofile} and/or sitewide \code{Rprofile}:\preformatted{if (!grepl("-Xrs", getOption("java.parameters", ""))) {
22
-  options(
23
-    "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
24
-  )
25
-}
26
-}
27
-}
28
-
29
-\references{
30
-\href{https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf}{Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide}
31
-}
32
-\author{
33
-Bob Rudis (bob@rud.is)
34
-}
35
-\keyword{internal}

+ 1
- 1
man/use_credentials.Rd View File

@@ -1,5 +1,5 @@
1 1
 % Generated by roxygen2: do not edit by hand
2
-% Please edit documentation in R/metis-lite-package.R
2
+% Please edit documentation in R/metis-package.R
3 3
 \name{use_credentials}
4 4
 \alias{use_credentials}
5 5
 \alias{read_credentials}

metis-lite.Rproj → metis.Rproj View File


+ 1
- 1
tests/test-all.R View File

@@ -1,2 +1,2 @@
1 1
 library(testthat)
2
-test_check("metis-lite")
2
+test_check("metis")

+ 0
- 6
tests/testthat/test-metis-lite.R View File

@@ -1,6 +0,0 @@
1
-context("basic functionality")
2
-test_that("we can do something", {
3
-
4
-  #expect_that(some_function(), is_a("data.frame"))
5
-
6
-})

+ 28
- 0
tests/testthat/test-metis.R View File

@@ -0,0 +1,28 @@
1
+context("Driver & queries work")
2
+
3
+skip_on_cran()
4
+
5
+drv <- metis::Athena()
6
+
7
+expect_is(drv, "AthenaDriver")
8
+
9
+dbConnect(
10
+  drv = drv,
11
+  schema_name = "sampledb",
12
+  provider = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
13
+  AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
14
+  s3_staging_dir = "s3://aws-athena-query-results-569593279821-us-east-1",
15
+) -> con
16
+
17
+expect_is(con, "AthenaConnection")
18
+
19
+expect_equal(dbListTables(con, schema="sampledb"), "elb_logs")
20
+
21
+expect_true(dbExistsTable(con, "elb_logs", schema="sampledb"))
22
+
23
+expect_true("url" %in% dbListFields(con, "elb_logs", "sampledb"))
24
+
25
+expect_is(
26
+  dbGetQuery(con, "SELECT * FROM sampledb.elb_logs LIMIT 10"),
27
+  "data.frame"
28
+)

Loading…
Cancel
Save