boB Rudis 9 months ago
parent
commit
cea670e412
No known key found for this signature in database
13 changed files with 140 additions and 169 deletions
  1. 1
    0
      .Rbuildignore
  2. 4
    6
      DESCRIPTION
  3. 2
    0
      LICENSE
  4. 21
    0
      LICENSE.md
  5. 3
    7
      NAMESPACE
  6. 12
    0
      R/a-utils.R
  7. 7
    113
      R/jdbc.r
  8. 5
    19
      R/metis-lite-package.R
  9. 71
    0
      R/z-dbGetQuery.R
  10. 2
    2
      man/Athena.Rd
  11. 2
    2
      man/dbConnect-AthenaDriver-method.Rd
  12. 1
    1
      man/dbGetQuery-AthenaConnection-character-method.Rd
  13. 9
    19
      man/metis.lite.Rd

+ 1
- 0
.Rbuildignore View File

@@ -1,3 +1,4 @@
1
+^LICENSE\.md$
1 2
 ^.*\.Rproj$
2 3
 ^\.Rproj\.user$
3 4
 ^\.travis\.yml$

+ 4
- 6
DESCRIPTION View File

@@ -13,7 +13,7 @@ Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schema
13 13
     perform queries and retrieve query results. A lightweight 'RJDBC' implementation 
14 14
     is included along with an interface to the 'AWS' command-line utility.
15 15
 SystemRequirements: JDK 1.8+
16
-License: AGPL
16
+License: MIT + file LICENSE
17 17
 Suggests:
18 18
     testthat,
19 19
     covr
@@ -23,9 +23,7 @@ Depends:
23 23
 Imports:
24 24
     rJava,
25 25
     DBI,
26
-    dplyr,
27
-    readr,
28
-    aws.signature,
29
-    uuid,
30
-    jsonlite
26
+    bit64,
27
+    dbplyr,
28
+    aws.signature
31 29
 RoxygenNote: 6.1.1

+ 2
- 0
LICENSE View File

@@ -0,0 +1,2 @@
1
+YEAR: 2019
2
+COPYRIGHT HOLDER: Bob Rudis

+ 21
- 0
LICENSE.md View File

@@ -0,0 +1,21 @@
1
+# MIT License
2
+
3
+Copyright (c) 2019 Bob Rudis
4
+
5
+Permission is hereby granted, free of charge, to any person obtaining a copy
6
+of this software and associated documentation files (the "Software"), to deal
7
+in the Software without restriction, including without limitation the rights
8
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+copies of the Software, and to permit persons to whom the Software is
10
+furnished to do so, subject to the following conditions:
11
+
12
+The above copyright notice and this permission notice shall be included in all
13
+copies or substantial portions of the Software.
14
+
15
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+SOFTWARE.

+ 3
- 7
NAMESPACE View File

@@ -1,10 +1,10 @@
1 1
 # Generated by roxygen2: do not edit by hand
2 2
 
3
-S3method(db_data_type,AthenaConnection)
4
-S3method(sql_translate_env,AthenaConnection)
5 3
 export(Athena)
6 4
 export(athena_connect)
5
+export(db_data_type.AthenaConnection)
7 6
 export(read_credentials)
7
+export(sql_translate_env.AthenaConnection)
8 8
 export(use_credentials)
9 9
 exportClasses(AthenaConnection)
10 10
 exportClasses(AthenaDriver)
@@ -18,11 +18,7 @@ exportMethods(dbReadTable)
18 18
 exportMethods(dbSendQuery)
19 19
 import(DBI)
20 20
 import(RJDBC)
21
-import(dplyr)
21
+import(bit64)
22 22
 importFrom(aws.signature,read_credentials)
23 23
 importFrom(aws.signature,use_credentials)
24
-importFrom(jsonlite,fromJSON)
25 24
 importFrom(rJava,.jcall)
26
-importFrom(readr,type_convert)
27
-importFrom(sys,exec_internal)
28
-importFrom(uuid,UUIDgenerate)

+ 12
- 0
R/a-utils.R View File

@@ -0,0 +1,12 @@
1
+set_names <- function (object = nm, nm) {
2
+  names(object) <- nm
3
+  object
4
+}
5
+
6
+as_date <- function(x) {
7
+  as.Date(x, origin = "1970-01-01")
8
+}
9
+
10
+as_posixct <- function(x) {
11
+  as.POSIXct(x, origin = "1970-01-01 00:00:00")
12
+}

+ 7
- 113
R/jdbc.r View File

@@ -1,31 +1,9 @@
1
-.jt <- list(
2
-  BIT = -7,
3
-  TINYINT = -6,
4
-  BIGINT = -5,
5
-  LONGVARBINARY = -4,
6
-  VARBINARY = -3,
7
-  BINARY = -2,
8
-  LONGVARCHAR = -1,
9
-  .NULL = 0,
10
-  CHAR = 1,
11
-  NUMERIC = 2,
12
-  DECIMAL = 3,
13
-  INTEGER = 4,
14
-  SMALLINT = 5,
15
-  FLOAT = 6,
16
-  REAL = 7,
17
-  DOUBLE = 8,
18
-  VARCHAR = 12,
19
-  DATE = 91,
20
-  TIME = 92,
21
-  TIMESTAMP = 93,
22
-  OTHER = 1111
23
-)
24
-
25
-stats::setNames(
1
+structure(
26 2
   0:6,
27
-  c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE")
28
-) -> .ll_trans
3
+  .Names = c(
4
+    "OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"
5
+  )
6
+)-> .ll_trans
29 7
 
30 8
 #' AthenaJDBC
31 9
 #'
@@ -85,10 +63,8 @@ setMethod(
85 63
     max_error_retries = 10,
86 64
     connection_timeout = 10000,
87 65
     socket_timeout = 10000,
88
-    # retry_base_delay = 100,
89
-    # retry_max_backoff_time = 1000,
90
-    log_path,
91
-    log_level,
66
+    log_path = "",
67
+    log_level = 0,
92 68
     ...) {
93 69
 
94 70
     conn_string = sprintf(
@@ -105,8 +81,6 @@ setMethod(
105 81
       MaxErrorRetry = max_error_retries,
106 82
       ConnectTimeout = connection_timeout,
107 83
       SocketTimeout = socket_timeout,
108
-      # retry_base_delay = retry_base_delay,
109
-      # retry_max_backoff_time = retry_max_backoff_time,
110 84
       LogPath = log_path,
111 85
       LogLevel = log_level,
112 86
       AwsCredentialsProviderClass = provider,
@@ -149,86 +123,6 @@ setMethod(
149 123
 #' AthenaJDBC
150 124
 #'
151 125
 #' @param conn Athena connection
152
-#' @param statement SQL statement
153
-#' @param ... unused
154
-#' @importFrom rJava .jcall
155
-#' @export
156
-setMethod(
157
-
158
-  "dbGetQuery",
159
-  signature(conn="AthenaConnection", statement="character"),
160
-
161
-  definition = function(conn, statement, type_convert=FALSE, ...) {
162
-
163
-    r <- dbSendQuery(conn, statement, ...)
164
-
165
-    on.exit(.jcall(r@stat, "V", "close"))
166
-
167
-    structure(
168
-      list(
169
-        cols = list(),
170
-        default = structure(list(), class = c("collector_guess", "collector"))
171
-      ),
172
-      class = "col_spec"
173
-    ) -> l
174
-
175
-    cols <- .jcall(r@md, "I", "getColumnCount")
176
-    nms <- c()
177
-
178
-    cts <- rep(0L, cols)
179
-    for (i in 1:cols) {
180
-      ct <- .jcall(r@md, "I", "getColumnType", i)
181
-      if (ct == .jt$CHAR) {
182
-        l$cols[[i]] <- col_character()
183
-      } else if (ct == .jt$NUMERIC) {
184
-        l$cols[[i]] <- col_double()
185
-      } else if (ct == .jt$DECIMAL) {
186
-        l$cols[[i]] <- col_double()
187
-      } else if (ct == .jt$INTEGER) {
188
-        l$cols[[i]] <- col_integer()
189
-      } else if (ct == .jt$SMALLINT) {
190
-        l$cols[[i]] <- col_integer()
191
-      } else if (ct == .jt$TINYINT) {
192
-        l$cols[[i]] <- col_integer()
193
-      } else if (ct == .jt$BIGINT) {
194
-        l$cols[[i]] <- col_double()
195
-      } else if (ct == .jt$FLOAT) {
196
-        l$cols[[i]] <- col_double()
197
-      } else if (ct == .jt$REAL) {
198
-        l$cols[[i]] <- col_double()
199
-      } else if (ct == .jt$DOUBLE) {
200
-        l$cols[[i]] <- col_double()
201
-      } else if (ct == .jt$VARCHAR) {
202
-        l$cols[[i]] <- col_character()
203
-      } else if (ct == .jt$DATE) {
204
-        l$cols[[i]] <- col_date()
205
-      } else if (ct == .jt$TIME) {
206
-        l$cols[[i]] <- col_time()
207
-      } else if (ct == .jt$TIMESTAMP) {
208
-        l$cols[[i]] <- col_datetime()
209
-      } else if (ct == .jt$BIT) {
210
-        l$cols[[i]] <- col_logical()
211
-      } else {
212
-        l$cols[[i]] <- col_character()
213
-      }
214
-      nms[i] <- .jcall(r@md, "S", "getColumnLabel", i)
215
-    }
216
-
217
-    l$cols <- stats::setNames(l$cols, nms)
218
-
219
-    res <- fetch(r, -1, block=1000)
220
-    res <- readr::type_convert(res, col_types = l)
221
-
222
-    class(res) <- c("tbl_df", "tbl", "data.frame")
223
-
224
-    res
225
-
226
-  }
227
-
228
-)
229
-#' AthenaJDBC
230
-#'
231
-#' @param conn Athena connection
232 126
 #' @param pattern table name pattern
233 127
 #' @param schema Athena schema name
234 128
 #' @param ... unused

+ 5
- 19
R/metis-lite-package.R View File

@@ -10,16 +10,7 @@
10 10
 #'
11 11
 #' Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
12 12
 #' Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid
13
-#' signals being passed on to the JVM owner. That has to be done _#' before_ `rJava` is
14
-#' loaded so you either need to remember to put it at the top of all scripts _or_ stick this
15
-#' in your local #' `~/.Rprofile` and/or sitewide `Rprofile`:
16
-#'
17
-#'
18
-#' @section IMPORTANT:
19
-#'
20
-#' Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
21
-#' Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid
22
-#' signals being passed on to the JVM owner. That has to be done _#' before_ `rJava` is
13
+#' signals being passed on to the JVM owner. That has to be done _before_ `rJava` is
23 14
 #' loaded so you either need to remember to put it at the top of all scripts _or_ stick this
24 15
 #' in your local #' `~/.Rprofile` and/or sitewide `Rprofile`:
25 16
 #'
@@ -30,24 +21,19 @@
30 21
 #'       )
31 22
 #'     }
32 23
 #'
24
+#' @md
33 25
 #' @encoding UTF-8
26
+#' @keywords internal
34 27
 #' @docType package
35 28
 #' @author Bob Rudis (bob@@rud.is)
36
-#' @import RJDBC
37
-#' @keywords internal
38
-#' @import DBI
39
-#' @import dplyr
40
-#' @importFrom jsonlite fromJSON
41
-#' @importFrom readr type_convert
42
-#' @importFrom uuid UUIDgenerate
43
-#' @importFrom sys exec_internal
44
-#' @importFrom aws.signature use_credentials read_credentials
29
+#' @import RJDBC DBI bit64 dbplyr
45 30
 NULL
46 31
 
47 32
 
48 33
 #' Use Credentials from .aws/credentials File
49 34
 #'
50 35
 #' @md
36
+#' @importFrom aws.signature use_credentials read_credentials
51 37
 #' @references [aws.signature::use_credentials()] /  [aws.signature::read_credentials()]
52 38
 #' @name use_credentials
53 39
 #' @rdname use_credentials

+ 71
- 0
R/z-dbGetQuery.R View File

@@ -0,0 +1,71 @@
1
+list(
2
+  "-7" = as.logical, # BIT
3
+  "-6" = as.integer, # TINYINT
4
+  "-5" = bit64::as.integer64, # BIGINT
5
+  "-4" = as.character, # LONGVARBINARY
6
+  "-3" = as.character, # VARBINARY
7
+  "-2" = as.character, # BINARY
8
+  "-1" = as.character, # LONGVARCHAR
9
+  "0" = as.character, # NULL
10
+  "1" = as.character, # CHAR
11
+  "2" = as.double, # NUMERIC
12
+  "3" = as.double, # DECIMAL
13
+  "4" = as.integer, # INTEGER
14
+  "5" = as.integer, # SMALLINT
15
+  "6" = as.double, # FLOAT
16
+  "7" = as.double, # REAL
17
+  "8" = as.double, # DOUBLE
18
+  "12" = as.character, # VARCHAR
19
+  "16" = as.logical, # BOOLEAN
20
+  "91" = as_date, # DATE
21
+  "92" = as.character, # TIME
22
+  "93" = as_posixct, # TIMESTAMP
23
+  "1111" = as.character # OTHER
24
+) -> .jdbc_converters
25
+
26
+#' AthenaJDBC
27
+#'
28
+#' @param conn Athena connection
29
+#' @param statement SQL statement
30
+#' @param ... unused
31
+#' @importFrom rJava .jcall
32
+#' @export
33
+setMethod(
34
+
35
+  "dbGetQuery",
36
+  signature(conn="AthenaConnection", statement="character"),
37
+
38
+  definition = function(conn, statement, type_convert=FALSE, ...) {
39
+
40
+    r <- dbSendQuery(conn, statement, ...)
41
+
42
+    on.exit(.jcall(r@stat, "V", "close"))
43
+
44
+    #message("dbGetQuery()")
45
+
46
+    nms <- c()
47
+    athena_type_convert <- list()
48
+
49
+    cols <- .jcall(r@md, "I", "getColumnCount")
50
+
51
+    for (i in 1:cols) {
52
+      ct <- as.character(.jcall(r@md, "I", "getColumnType", i))
53
+      athena_type_convert[[i]] <- .jdbc_converters[[ct]]
54
+      nms <- c(nms, .jcall(r@md, "S", "getColumnLabel", i))
55
+    }
56
+
57
+    athena_type_convert <- set_names(athena_type_convert, nms)
58
+
59
+    res <- fetch(r, -1, block = 1000)
60
+
61
+    for (nm in names(athena_type_convert)) {
62
+      res[[nm]] <- athena_type_convert[[nm]](res[[nm]])
63
+    }
64
+
65
+    class(res) <- c("tbl_df", "tbl", "data.frame")
66
+
67
+    res
68
+
69
+  }
70
+
71
+)

+ 2
- 2
man/Athena.Rd View File

@@ -8,9 +8,9 @@
8 8
 \usage{
9 9
 Athena(identifier.quote = "`")
10 10
 
11
-\method{db_data_type}{AthenaConnection}(con, fields, ...)
11
+db_data_type.AthenaConnection(con, fields, ...)
12 12
 
13
-\method{sql_translate_env}{AthenaConnection}(con)
13
+sql_translate_env.AthenaConnection(con)
14 14
 }
15 15
 \description{
16 16
 AthenaJDBC

+ 2
- 2
man/dbConnect-AthenaDriver-method.Rd View File

@@ -10,8 +10,8 @@
10 10
   region = "us-east-1",
11 11
   s3_staging_dir = Sys.getenv("AWS_S3_STAGING_DIR"),
12 12
   schema_name = "default", max_error_retries = 10,
13
-  connection_timeout = 10000, socket_timeout = 10000, log_path,
14
-  log_level, ...)
13
+  connection_timeout = 10000, socket_timeout = 10000, log_path = "",
14
+  log_level = 0, ...)
15 15
 }
16 16
 \arguments{
17 17
 \item{provider}{JDBC auth provider (ideally leave default)}

+ 1
- 1
man/dbGetQuery-AthenaConnection-character-method.Rd View File

@@ -1,5 +1,5 @@
1 1
 % Generated by roxygen2: do not edit by hand
2
-% Please edit documentation in R/jdbc.r
2
+% Please edit documentation in R/z-dbGetQuery.R
3 3
 \docType{methods}
4 4
 \name{dbGetQuery,AthenaConnection,character-method}
5 5
 \alias{dbGetQuery,AthenaConnection,character-method}

+ 9
- 19
man/metis.lite.Rd View File

@@ -15,25 +15,15 @@ is included along with an interface to the 'AWS' command-line utility.
15 15
 
16 16
 
17 17
 Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
18
-Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid
19
-signals being passed on to the JVM owner. That has to be done _#' before_ `rJava` is
20
-loaded so you either need to remember to put it at the top of all scripts _or_ stick this
21
-in your local #' `~/.Rprofile` and/or sitewide `Rprofile`:
22
-
23
-
24
-
25
-Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting
26
-Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid
27
-signals being passed on to the JVM owner. That has to be done _#' before_ `rJava` is
28
-loaded so you either need to remember to put it at the top of all scripts _or_ stick this
29
-in your local #' `~/.Rprofile` and/or sitewide `Rprofile`:
30
-
31
-
32
-    if (!grepl("-Xrs", getOption("java.parameters", ""))) {
33
-      options(
34
-        "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
35
-      )
36
-    }
18
+Athena JDBC calls crash the R #' interpreter. You need to set the \code{-Xrs} option to avoid
19
+signals being passed on to the JVM owner. That has to be done \emph{before} \code{rJava} is
20
+loaded so you either need to remember to put it at the top of all scripts \emph{or} stick this
21
+in your local #' \code{~/.Rprofile} and/or sitewide \code{Rprofile}:\preformatted{if (!grepl("-Xrs", getOption("java.parameters", ""))) {
22
+  options(
23
+    "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs")
24
+  )
25
+}
26
+}
37 27
 }
38 28
 
39 29
 \author{

Loading…
Cancel
Save