Browse Source

initial commit

master
boB Rudis 2 months ago
parent
commit
9d81c51478
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
14 changed files with 682 additions and 13 deletions
  1. +8
    -4
      DESCRIPTION
  2. +2
    -0
      LICENSE
  3. +21
    -0
      LICENSE.md
  4. +4
    -1
      NAMESPACE
  5. +48
    -0
      R/h2c.R
  6. +7
    -4
      R/httr2curl-package.R
  7. +11
    -0
      R/utils-pipe.R
  8. +14
    -0
      R/utils.R
  9. +16
    -2
      README.Rmd
  10. +80
    -0
      README.md
  11. +426
    -0
      inst/bin/h2c.pl
  12. +29
    -0
      man/h2c.Rd
  13. +4
    -2
      man/httr2curl.Rd
  14. +12
    -0
      man/pipe.Rd

+ 8
- 4
DESCRIPTION View File

@ -1,6 +1,6 @@
Package: httr2curl
Type: Package
Title: httr2curl Title Goes Here Otherwise CRAN Checks Fail
Title: Turn 'httr' Web Requests Into 'curl' Command Line Calls
Version: 0.1.0
Date: 2021-01-27
Authors@R: c(
@ -8,17 +8,21 @@ Authors@R: c(
comment = c(ORCID = "0000-0001-5670-2640"))
)
Maintainer: Bob Rudis <bob@rud.is>
Description: A good description goes here otherwise CRAN checks fail.
Description: When provided with an 'httr' "VERB" call, the request will be made
and 'libcurl' headers will be captured and post-processed to provide a working
'curl' command line call.
URL: https://git.rud.is/hrbrmstr/httr2curl
BugReports: https://git.rud.is/hrbrmstr/httr2curl/issues
Encoding: UTF-8
License: AGPL
License: MIT + file LICENSE
SystemRequirements: perl
Suggests:
covr, tinytest
Depends:
R (>= 3.6.0)
Imports:
httr,
jsonlite
processx,
magrittr
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.1.1

+ 2
- 0
LICENSE View File

@ -0,0 +1,2 @@
YEAR: 2021
COPYRIGHT HOLDER: Bob Rudis

+ 21
- 0
LICENSE.md View File

@ -0,0 +1,21 @@
# MIT License
Copyright (c) 2021 Bob Rudis
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

+ 4
- 1
NAMESPACE View File

@ -1,4 +1,7 @@
# Generated by roxygen2: do not edit by hand
export("%>%")
export(h2c)
import(httr)
importFrom(jsonlite,fromJSON)
import(processx)
importFrom(magrittr,"%>%")

+ 48
- 0
R/h2c.R View File

@ -0,0 +1,48 @@
#' Convert an 'httr' call to 'curl' command line
#'
#' @param complete_httr_verb_call wrap an `httr` `VERB` call with this function
#' and it will return the text of a working `curl` command line
#' @export
#' @examples \dontrun{
#' h2c(
#' httr::GET(
#' url = "https://rud.is/",
#' httr::user_agent(splashr::ua_apple_tv),
#' query = list(
#' a = "b",
#' c = 1
#' )
#' )
#' )
#' }
h2c <- function(complete_httr_verb_call) {
ƒ_call <- substitute(complete_httr_verb_call)
perl <- find_perl()
args <- system.file("bin", "h2c.pl", package = "httr2curl")
capture.output(
capture.output(
httr::with_verbose(
eval(ƒ_call)
), type = "message") -> res
) -> junk
out <- tempfile()
res[grepl("^->", res)] %>%
sub("^-> ", "", .) %>%
paste0(collapse = "\n") %>%
writeLines(out)
processx::run(
command = perl,
args = args,
stdin = out
) -> res
res$stdout
}

+ 7
- 4
R/httr2curl-package.R View File

@ -1,9 +1,12 @@
#' ...
#'
#' Turn 'httr' Web Requests Into 'curl' Command Line Calls
#'
#' When provided with an 'httr' "VERB" call, the request will be made
#' and 'libcurl' headers will be captured and post-processed to provide a working
#' 'curl' command line call.
#'
#' @md
#' @name httr2curl
#' @keywords internal
#' @author Bob Rudis (bob@@rud.is)
#' @import httr
#' @importFrom jsonlite fromJSON
#' @import httr processx
"_PACKAGE"

+ 11
- 0
R/utils-pipe.R View File

@ -0,0 +1,11 @@
#' Pipe operator
#'
#' See \code{magrittr::\link[magrittr:pipe]{\%>\%}} for details.
#'
#' @name %>%
#' @rdname pipe
#' @keywords internal
#' @export
#' @importFrom magrittr %>%
#' @usage lhs \%>\% rhs
NULL

+ 14
- 0
R/utils.R View File

@ -0,0 +1,14 @@
find_perl <- function() {
perl <- Sys.which("perl")
if (perl == "") {
stop(
"Cannot find 'perl'. httr2curl requires perl to be installed and on the PATH.",
call. = FALSE
)
}
return(perl)
}

+ 16
- 2
README.Rmd View File

@ -3,6 +3,7 @@ output: rmarkdown::github_document
editor_options:
chunk_output_type: console
---
```{r pkg-knitr-opts, include=FALSE}
hrbrpkghelpr::global_opts()
```
@ -36,7 +37,19 @@ library(httr2curl)
# current version
packageVersion("httr2curl")
```
```{r ex01}
h2c(
httr::GET(
url = "https://rud.is/",
httr::user_agent(splashr::ua_apple_tv),
query = list(
a = "b",
c = 1
)
)
)
```
## httr2curl Metrics
@ -47,5 +60,6 @@ cloc::cloc_pkg_md()
## Code of Conduct
Please note that this project is released with a Contributor Code of Conduct.
By participating in this project you agree to abide by its terms.
Please note that this project is released with a Contributor Code of
Conduct. By participating in this project you agree to abide by its
terms.

+ 80
- 0
README.md View File

@ -0,0 +1,80 @@
[![Project Status: Active – The project has reached a stable, usable
state and is being actively
developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
[![Signed
by](https://img.shields.io/badge/Keybase-Verified-brightgreen.svg)](https://keybase.io/hrbrmstr)
![Signed commit
%](https://img.shields.io/badge/Signed_Commits-100%25-lightgrey.svg)
[![R-CMD-check](https://github.com/hrbrmstr/httr2curl/workflows/R-CMD-check/badge.svg)](https://github.com/hrbrmstr/httr2curl/actions?query=workflow%3AR-CMD-check)
[![Linux build
Status](https://travis-ci.org/hrbrmstr/httr2curl.svg?branch=master)](https://travis-ci.org/hrbrmstr/httr2curl)
![Minimal R
Version](https://img.shields.io/badge/R%3E%3D-3.6.0-blue.svg)
![License](https://img.shields.io/badge/License-MIT-blue.svg)
# httr2curl
Turn ‘httr’ Web Requests Into ‘curl’ Command Line Calls
## Description
When provided with an ‘httr’ “VERB” call, the request will be made and
‘libcurl’ headers will be captured and post-processed to provide a
working ‘curl’ command line call.
## What’s Inside The Tin
The following functions are implemented:
- `h2c`: Convert an ‘httr’ call to ‘curl’ command line
## Installation
``` r
remotes::install_git("https://git.rud.is/hrbrmstr/httr2curl.git")
```
NOTE: To use the ‘remotes’ install options you will need to have the
[{remotes} package](https://github.com/r-lib/remotes) installed.
## Usage
``` r
library(httr2curl)
# current version
packageVersion("httr2curl")
## [1] '0.1.0'
```
``` r
h2c(
httr::GET(
url = "https://rud.is/",
httr::user_agent(splashr::ua_apple_tv),
query = list(
a = "b",
c = 1
)
)
)
## [1] "curl --http2 --header \"Accept: application/json, text/xml, application/xml, */*\" --compressed --user-agent \"AppleTV6,2/11.1\" \"https://rud.is/?a=b&c=1\"\n"
```
## httr2curl Metrics
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) |
|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:|
| R | 5 | 0.36 | 37 | 0.24 | 15 | 0.23 | 38 | 0.27 |
| YAML | 1 | 0.07 | 22 | 0.14 | 2 | 0.03 | 2 | 0.01 |
| Rmd | 1 | 0.07 | 18 | 0.12 | 16 | 0.24 | 31 | 0.22 |
| SUM | 7 | 0.50 | 77 | 0.50 | 33 | 0.50 | 71 | 0.50 |
clock Package Metrics for httr2curl
## Code of Conduct
Please note that this project is released with a Contributor Code of
Conduct. By participating in this project you agree to abide by its
terms.

+ 426
- 0
inst/bin/h2c.pl View File

@ -0,0 +1,426 @@
#!/usr/bin/perl
use MIME::Base64;
sub usage {
print "h2c.pl [options] < file \n",
" -a Allow curl's default headers\n",
" -d Output man page HTML links after command line\n",
" -h Show short help\n",
" -H Output HTTP generated URLs instead\n",
" -i Ignore HTTP version\n",
" --libcurl Output libcurl code instead\n",
" -n Output notes after command line\n",
" -s Use short command line options\n",
" -v Add a verbose option to the command line\n";
exit;
}
sub manpage {
my ($p, $n, $desc) = @_;
if(!$n) {
$n = $p;
}
return sprintf("%s;%s;$desc", $p, $n);
}
my $usesamehttpversion = 1;
my $disableheadersnotseen = 1;
my $shellcompatible = 1; # may not been windows command prompt compat
my $uselongoptions = 1; # instead of short
my $uselibcurl = 0; # --libcurl
my $usehttp = 0;
while($ARGV[0]) {
if(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) {
usage();
}
elsif($ARGV[0] eq "-a") {
$disableheadersnotseen = 0;
shift @ARGV;
}
elsif($ARGV[0] eq "-d") {
$usedocs = 1;
shift @ARGV;
}
elsif($ARGV[0] eq "-H") {
$usehttp = 1;
shift @ARGV;
}
elsif($ARGV[0] eq "-i") {
$usesamehttpversion = 0;
shift @ARGV;
}
elsif($ARGV[0] eq "--libcurl") {
$uselibcurl = 1;
shift @ARGV;
}
elsif($ARGV[0] eq "-n") {
$usenotes = 1;
shift @ARGV;
}
elsif($ARGV[0] eq "-s") {
$uselongoptions = 0;
shift @ARGV;
}
elsif($ARGV[0] eq "-v") {
$useverbose = 1;
shift @ARGV;
}
else {
usage();
}
}
my $state; # 0 is request-line, 1-headers, 2-body
my $line = 1;
while(<STDIN>) {
my $l = $_;
# discard CRs completely
$l =~ s/\r//g;
if(!$state) {
chomp $l;
if($l =~ /([^ ]*) +(.*) +(HTTP\/.*)/) {
$method = $1;
$path = $2;
$http = $3;
}
else {
$error="bad request-line";
last;
}
$state++;
}
elsif(1 == $state) {
chomp $l;
if($l =~ /([^:]*): *(.*)/) {
$header{lc($1)}=$2;
$exactcase{lc($1)}=$1; # to allow us to use it as read
}
elsif(length($l)<2) {
# body time
$state++;
}
else {
$error="illegal HTTP header on line $line";
last;
}
}
elsif(2 == $state) {
push @body, $l;
}
$line++;
}
if(!$header{'host'}) {
$error = "No Host: header makes it impossible to tell URL\n";
}
error:
if($error) {
print "Error: $error\n";
exit;
}
if($uselongoptions) {
$opt_data = "--data";
$opt_request = "--request";
$opt_head = "--head";
$opt_header = "--header";
$opt_user_agent = "--user-agent";
$opt_cookie = "--cookie";
$opt_verbose = "--verbose";
$opt_form = "--form";
$opt_user = "--user";
}
else {
$opt_data = "-d";
$opt_request = "-X";
$opt_head = "-I";
$opt_header = "-H";
$opt_user_agent = "-A";
$opt_cookie = "-b";
$opt_verbose = "-v";
$opt_form = "-F";
$opt_user = "-u";
}
my $httpver="";
my $disabledheaders="";
my $addedheaders="";
if($header{"content-type"} =~ /^multipart\/form-data;/) {
# multipart formpost, this is special
my $type = $header{"content-type"};
my $boundary = $type;
$boundary =~ s/.*boundary=(.*)/$1/;
my $inbound = $body[0];
chomp $inbound;
# a body MUST start with dash-dash-boundary
if("--$boundary" ne $inbound) {
$error = "unexpected multipart format";
goto error;
}
my $bline=1;
my %fheader;
my $fstate = 0;
my @fbody;
while($body[$bline]) {
my $l = $body[$bline];
if(0 == $fstate) {
# headers
chomp $l;
if($l =~ /([^:]*): *(.*)/) {
$fheader{lc($1)}=$2;
}
elsif(length($l)<2) {
# body time
$fstate++;
}
}
elsif($fstate) {
if($l =~ /^--$boundary/) {
# end of this part
my $cd = $fheader{'content-disposition'};
if(!$cd) {
$error = "multi-part without Content-Disposition: header!";
goto error;
}
# Content-Disposition: form-data; name="name"
# Content-Disposition: form-data; name="file"; filename="README.md"
if($cd =~ /^form-data; name=([^;]*)[;]? *(.*)/i) {
my ($n, $f)=($1, $2);
# name is with or without quotes
$n =~ s/\"//g;
if($f =~ /^filename=(.*)/) {
# filename is with or without quotes
$f = $1;
$f =~ s/\"//g;
}
if(!$multipart) {
push @docs, manpage("-F", $opt_form, "send a multipart formpost");
}
if(!$f) {
my $fbody = join("", @fbody);
$fbody =~ s/[ \n\r]+\z//g;
$fbody =~ s/([\\\$\"\'\`])/\\$1/g;
$multipart .= "$opt_form $n=\"$fbody\" ";
@fbody="";
}
else {
# file name was present
$multipart .= "$opt_form $n=\@$f ";
}
}
$fstate = 0;
%fheader = 0;
$bline++;
next;
}
push @fbody, $l;
}
$bline++;
}
if($body[$bline-1] !~ /^--$boundary--/) {
print STDERR "bad last line?";
}
$header{"content-type"} = ""; # blank it
$do_multipart = 1;
}
elsif(length(join("", @body))) {
# TODO: escape the body
my $esc = join("", @body);
chomp $esc; # trim the final newline
if($shellcompatible) {
$esc =~ s/\n/ /g; # turn newlines into space!
$esc =~ s/\"/\\"/g; # escape double quotes
if(!$unixescaped) {
push @notes, "uses quotes suitable for *nix command lines";
$unixescaped++;
}
}
$usebody= sprintf("--data-binary \"%s\" ", $esc);
push @docs, manpage("--data-binary", "", "send this string as a body with POST");
}
if(uc($method) eq "HEAD") {
$usemethod = "$opt_head ";
push @docs, manpage("-I", $opt_head, "send a HEAD request");
}
elsif(uc($method) eq "POST") {
if(!$usebody && !$do_multipart) {
$usebody= sprintf("$opt_data \"\" ");
push @docs, manpage("-d", $opt_data, "send this string as a body with POST");
}
}
elsif(uc($method) eq "PUT") {
if(!$usebody) {
$usebody= sprintf("$opt_data \"\" ");
push @docs, manpage("-d", $opt_data, "send this string as a body with POST");
}
$usebody .= "$opt_request PUT ";
push @docs, manpage("-X", $opt_request, "replace the request method with this string");
}
elsif(uc($method) eq "OPTIONS") {
$usemethod .= "$opt_request OPTIONS ";
if($path !~ /^\//) {
# very special case
$requesttarget="--request-target \"$path\" ";
push @docs, manpage("--request-target", "",
"specify request target to use instead of using the URL's");
$path = "";
}
}
elsif(uc($method) ne "GET") {
$error = "unsupported HTTP method $method";
goto error;
}
if($usebody) {
# body is set, handle the content-type
if(!$header{"content-type"}) {
$disabledheaders .= "$opt_header Content-Type: ";
}
elsif(lc($header{"content-type"}) ne
"application/x-www-form-urlencoded") {
# custom
$ignore_contenttype = 1;
$addedheaders .= sprintf("$opt_header \"Content-Type: %s\" ",
$header{"content-type"});
}
elsif((lc($header{"content-type"}) eq
"application/x-www-form-urlencoded") && !$do_multipart &&
($method eq "POST")) {
# default for normal POST
$ignore_contenttype = 1;
}
}
if($usesamehttpversion) {
if(uc($http) eq "HTTP/1.1") {
$httpver = "--http1.1 ";
push @docs, manpage("--http1.1", "", "use HTTP protocol version 1.1");
}
elsif(uc($http) eq "HTTP/2") {
$httpver = "--http2 ";
push @docs, manpage("--http2", "", "use HTTP protocol version 2");
}
else {
$error = "unsupported HTTP version $http";
goto error;
}
}
if($disableheadersnotseen) {
if(!$header{'accept'}) {
$disabledheaders .= "$opt_header Accept: ";
}
if(!$header{'user-agent'}) {
$disabledheaders .= "$opt_header User-Agent: ";
}
}
if($do_multipart) {
if(!$header{lc("expect")}) {
# no expect header, disable it for us too since curl -F defaults to
# Expect: 100-continue
$disabledheaders .= "$opt_header Expect: ";
}
}
# go through the headers alphabetically just to make the order fixed
foreach my $h (sort keys %header) {
if(lc($h) eq "host") {
# We use Host: for the URL creation
}
elsif((lc($h) eq "authorization") &&
($header{'authorization'} =~ /^Basic (.*)/)) {
my $decoded = decode_base64($1);
$addedheaders .= sprintf("%s \"%s\" ", $opt_user, $decoded);
push @docs, manpage("-u", $opt_user, "use this user and password for Basic auth");
}
elsif(lc($h) eq "expect") {
# let curl do expect on its own
}
elsif(lc($h) eq "content-type" &&
($do_multipart || $ignore_contenttype)) {
# skip this for multipart
}
elsif(($h eq "accept-encoding") &&
($header{$h} =~ /gzip/)) {
push @docs, manpage("--compressed", "", "request a compressed response");
$addedheaders .= "--compressed ";
}
elsif((lc($h) eq "accept") &&
($header{"accept"} eq "*/*")) {
# ignore if set to */* as that's a curl default
}
elsif(lc($h) eq "content-length") {
# we don't set custom size, just usebody
}
else {
$exact = $exactcase{$h};
my $opt = sprintf("$opt_header \"%s: ", $exact);
if(lc($h) eq "user-agent") {
$opt = "$opt_user_agent \"";
push @docs, manpage("-A", $opt_user_agent, "use this custom User-Agent request header");
}
elsif(lc($h) eq "cookie") {
$opt = "$opt_cookie \"";
push @docs, manpage("-b", $opt_cookie, "pass on this custom Cookie: request header");
}
$addedheaders .= sprintf("%s%s\" ", $opt, $header{$h});
}
}
if($path =~ /[ &?]/) {
$url = sprintf "\"%s://%s%s\"", $usehttp ? "http" : "https", $header{'host'}, $path;
}
else {
$url = sprintf "%s://%s%s", $usehttp ? "http" : "https", $header{'host'}, $path;
}
if($disabledheaders || $addedheaders) {
push @docs, manpage("-H", $opt_header, "add, replace or remove HTTP headers from the request");
}
if($useverbose) {
$useverbose = "$opt_verbose ";
push @docs, manpage("-v", $opt_verbose, "show verbose output");
}
# This adds the -x option to prevent a curl request to actually go out to any
# remote server
my $lib="--libcurl - -x localhost:0 " if($uselibcurl);
my $curlcmd = sprintf "curl ${useverbose}${usemethod}${httpver}${disabledheaders}${addedheaders}${usebody}${multipart}${requesttarget}${lib}${url}";
if($uselibcurl) {
# this actually runs curl which will fail to connect so ignore errors
open(C, "$curlcmd 2>/dev/null|");
while(<C>) {
# skip CURLOPT_PROXY since that's only used to avoid network
if($_ !~ /CURLOPT_PROXY, /) {
print $_;
}
}
close(C);
}
else {
print "$curlcmd\n";
}
if($usenotes) {
print "---\n";
foreach my $n (@notes) {
print "$n\n";
}
}
if($usedocs) {
print "---\n";
foreach my $d (@docs) {
print "$d\n";
}
}

+ 29
- 0
man/h2c.Rd View File

@ -0,0 +1,29 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/h2c.R
\name{h2c}
\alias{h2c}
\title{Convert an 'httr' call to 'curl' command line}
\usage{
h2c(complete_httr_verb_call)
}
\arguments{
\item{complete_httr_verb_call}{wrap an \code{httr} \code{VERB} call with this function
and it will return the text of a working \code{curl} command line}
}
\description{
Convert an 'httr' call to 'curl' command line
}
\examples{
\dontrun{
h2c(
httr::GET(
url = "https://rud.is/",
httr::user_agent(splashr::ua_apple_tv),
query = list(
a = "b",
c = 1
)
)
)
}
}

+ 4
- 2
man/httr2curl.Rd View File

@ -4,9 +4,11 @@
\name{httr2curl}
\alias{httr2curl}
\alias{httr2curl-package}
\title{...}
\title{Turn 'httr' Web Requests Into 'curl' Command Line Calls}
\description{
A good description goes here otherwise CRAN checks fail.
When provided with an 'httr' "VERB" call, the request will be made
and 'libcurl' headers will be captured and post-processed to provide a working
'curl' command line call.
}
\seealso{
Useful links:


+ 12
- 0
man/pipe.Rd View File

@ -0,0 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/utils-pipe.R
\name{\%>\%}
\alias{\%>\%}
\title{Pipe operator}
\usage{
lhs \%>\% rhs
}
\description{
See \code{magrittr::\link[magrittr:pipe]{\%>\%}} for details.
}
\keyword{internal}

Loading…
Cancel
Save