Merge branch 'main' into gpg-encryption-support
This commit is contained in:
Andrea Spacca
33
.github/build/friendly-filenames.json
vendored
Normal file
33
.github/build/friendly-filenames.json
vendored
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"android-arm64": { "friendlyName": "android-arm64-v8a" },
|
||||||
|
"darwin-amd64": { "friendlyName": "darwin-amd64" },
|
||||||
|
"darwin-arm64": { "friendlyName": "darwin-arm64" },
|
||||||
|
"dragonfly-amd64": { "friendlyName": "dragonfly-amd64" },
|
||||||
|
"freebsd-386": { "friendlyName": "freebsd-386" },
|
||||||
|
"freebsd-amd64": { "friendlyName": "freebsd-amd64" },
|
||||||
|
"freebsd-arm64": { "friendlyName": "freebsd-arm64-v8a" },
|
||||||
|
"freebsd-arm7": { "friendlyName": "freebsd-arm32-v7a" },
|
||||||
|
"linux-386": { "friendlyName": "linux-386" },
|
||||||
|
"linux-amd64": { "friendlyName": "linux-amd64" },
|
||||||
|
"linux-arm5": { "friendlyName": "linux-arm32-v5" },
|
||||||
|
"linux-arm64": { "friendlyName": "linux-arm64-v8a" },
|
||||||
|
"linux-arm6": { "friendlyName": "linux-arm32-v6" },
|
||||||
|
"linux-arm7": { "friendlyName": "linux-armv7" },
|
||||||
|
"linux-mips64le": { "friendlyName": "linux-mips64le" },
|
||||||
|
"linux-mips64": { "friendlyName": "linux-mips64" },
|
||||||
|
"linux-mipslesoftfloat": { "friendlyName": "linux-mips32le-softfloat" },
|
||||||
|
"linux-mipsle": { "friendlyName": "linux-mips32le" },
|
||||||
|
"linux-mipssoftfloat": { "friendlyName": "linux-mips32-softfloat" },
|
||||||
|
"linux-mips": { "friendlyName": "linux-mips32" },
|
||||||
|
"linux-ppc64le": { "friendlyName": "linux-ppc64le" },
|
||||||
|
"linux-ppc64": { "friendlyName": "linux-ppc64" },
|
||||||
|
"linux-riscv64": { "friendlyName": "linux-riscv64" },
|
||||||
|
"linux-s390x": { "friendlyName": "linux-s390x" },
|
||||||
|
"openbsd-386": { "friendlyName": "openbsd-386" },
|
||||||
|
"openbsd-amd64": { "friendlyName": "openbsd-amd64" },
|
||||||
|
"openbsd-arm64": { "friendlyName": "openbsd-arm64-v8a" },
|
||||||
|
"openbsd-arm7": { "friendlyName": "openbsd-arm32-v7a" },
|
||||||
|
"windows-386": { "friendlyName": "windows-386" },
|
||||||
|
"windows-amd64": { "friendlyName": "windows-amd64" },
|
||||||
|
"windows-arm7": { "friendlyName": "windows-arm32-v7a" }
|
||||||
|
}
|
||||||
89
.github/workflows/build-docker-images.yml
vendored
Normal file
89
.github/workflows/build-docker-images.yml
vendored
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
name: deploy multi-architecture Docker images for transfer.sh with buildx
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: '0 0 * * *' # everyday at midnight UTC
|
||||||
|
pull_request:
|
||||||
|
branches: main
|
||||||
|
push:
|
||||||
|
branches: main
|
||||||
|
tags:
|
||||||
|
- v*
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
buildx:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
-
|
||||||
|
name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
-
|
||||||
|
name: Prepare
|
||||||
|
id: prepare
|
||||||
|
run: |
|
||||||
|
DOCKER_IMAGE=dutchcoders/transfer.sh
|
||||||
|
DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64,linux/386
|
||||||
|
VERSION=edge
|
||||||
|
|
||||||
|
if [[ $GITHUB_REF == refs/tags/* ]]; then
|
||||||
|
VERSION=v${GITHUB_REF#refs/tags/v}
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||||
|
VERSION=nightly
|
||||||
|
fi
|
||||||
|
|
||||||
|
TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
|
||||||
|
|
||||||
|
if [ $VERSION = edge -o $VERSION = nightly ]; then
|
||||||
|
TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ::set-output name=docker_image::${DOCKER_IMAGE}
|
||||||
|
echo ::set-output name=version::${VERSION}
|
||||||
|
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \
|
||||||
|
--build-arg VERSION=${VERSION} \
|
||||||
|
--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
|
||||||
|
--build-arg VCS_REF=${GITHUB_SHA::8} \
|
||||||
|
${TAGS} .
|
||||||
|
-
|
||||||
|
name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v1
|
||||||
|
with:
|
||||||
|
platforms: all
|
||||||
|
-
|
||||||
|
name: Set up Docker Buildx
|
||||||
|
id: buildx
|
||||||
|
uses: docker/setup-buildx-action@v1
|
||||||
|
with:
|
||||||
|
version: latest
|
||||||
|
-
|
||||||
|
name: Available platforms
|
||||||
|
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||||
|
-
|
||||||
|
name: Docker Buildx (build)
|
||||||
|
run: |
|
||||||
|
docker buildx build --no-cache --pull --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }}
|
||||||
|
-
|
||||||
|
name: Docker Login
|
||||||
|
if: success() && github.event_name != 'pull_request'
|
||||||
|
env:
|
||||||
|
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
run: |
|
||||||
|
echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin
|
||||||
|
-
|
||||||
|
name: Docker Buildx (push)
|
||||||
|
if: success() && github.event_name != 'pull_request'
|
||||||
|
run: |
|
||||||
|
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
|
||||||
|
-
|
||||||
|
name: Docker Check Manifest
|
||||||
|
if: always() && github.event_name != 'pull_request'
|
||||||
|
run: |
|
||||||
|
docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}
|
||||||
|
-
|
||||||
|
name: Clear
|
||||||
|
if: always() && github.event_name != 'pull_request'
|
||||||
|
run: |
|
||||||
|
rm -f ${HOME}/.docker/config.json
|
||||||
171
.github/workflows/release.yml
vendored
Normal file
171
.github/workflows/release.yml
vendored
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
name: Build and Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
# Include amd64 on all platforms.
|
||||||
|
goos: [windows, freebsd, openbsd, linux, dragonfly, darwin]
|
||||||
|
goarch: [amd64, 386]
|
||||||
|
exclude:
|
||||||
|
# Exclude i386 on darwin and dragonfly.
|
||||||
|
- goarch: 386
|
||||||
|
goos: dragonfly
|
||||||
|
- goarch: 386
|
||||||
|
goos: darwin
|
||||||
|
include:
|
||||||
|
# BEIGIN MacOS ARM64
|
||||||
|
- goos: darwin
|
||||||
|
goarch: arm64
|
||||||
|
# END MacOS ARM64
|
||||||
|
# BEGIN Linux ARM 5 6 7
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm
|
||||||
|
goarm: 7
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm
|
||||||
|
goarm: 6
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm
|
||||||
|
goarm: 5
|
||||||
|
# END Linux ARM 5 6 7
|
||||||
|
# BEGIN Android ARM 8
|
||||||
|
- goos: android
|
||||||
|
goarch: arm64
|
||||||
|
# END Android ARM 8
|
||||||
|
# Windows ARM 7
|
||||||
|
- goos: windows
|
||||||
|
goarch: arm
|
||||||
|
goarm: 7
|
||||||
|
# BEGIN Other architectures
|
||||||
|
# BEGIN riscv64 & ARM64
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm64
|
||||||
|
- goos: linux
|
||||||
|
goarch: riscv64
|
||||||
|
# END riscv64 & ARM64
|
||||||
|
# BEGIN MIPS
|
||||||
|
- goos: linux
|
||||||
|
goarch: mips64
|
||||||
|
- goos: linux
|
||||||
|
goarch: mips64le
|
||||||
|
- goos: linux
|
||||||
|
goarch: mipsle
|
||||||
|
- goos: linux
|
||||||
|
goarch: mips
|
||||||
|
# END MIPS
|
||||||
|
# BEGIN PPC
|
||||||
|
- goos: linux
|
||||||
|
goarch: ppc64
|
||||||
|
- goos: linux
|
||||||
|
goarch: ppc64le
|
||||||
|
# END PPC
|
||||||
|
# BEGIN FreeBSD ARM
|
||||||
|
- goos: freebsd
|
||||||
|
goarch: arm64
|
||||||
|
- goos: freebsd
|
||||||
|
goarch: arm
|
||||||
|
goarm: 7
|
||||||
|
# END FreeBSD ARM
|
||||||
|
# BEGIN S390X
|
||||||
|
- goos: linux
|
||||||
|
goarch: s390x
|
||||||
|
# END S390X
|
||||||
|
# END Other architectures
|
||||||
|
# BEGIN OPENBSD ARM
|
||||||
|
- goos: openbsd
|
||||||
|
goarch: arm64
|
||||||
|
- goos: openbsd
|
||||||
|
goarch: arm
|
||||||
|
goarm: 7
|
||||||
|
# END OPENBSD ARM
|
||||||
|
fail-fast: false
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GOOS: ${{ matrix.goos }}
|
||||||
|
GOARCH: ${{ matrix.goarch }}
|
||||||
|
GOARM: ${{ matrix.goarm }}
|
||||||
|
CGO_ENABLED: 0
|
||||||
|
steps:
|
||||||
|
- name: Checkout codebase
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
|
- name: Show workflow information
|
||||||
|
id: get_filename
|
||||||
|
run: |
|
||||||
|
export _NAME=$(jq ".[\"$GOOS-$GOARCH$GOARM$GOMIPS\"].friendlyName" -r < .github/build/friendly-filenames.json)
|
||||||
|
echo "GOOS: $GOOS, GOARCH: $GOARCH, GOARM: $GOARM, GOMIPS: $GOMIPS, RELEASE_NAME: $_NAME"
|
||||||
|
echo "::set-output name=ASSET_NAME::$_NAME"
|
||||||
|
echo "::set-output name=GIT_TAG::${GITHUB_REF##*/}"
|
||||||
|
echo "ASSET_NAME=$_NAME" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v2
|
||||||
|
with:
|
||||||
|
go-version: ^1.17
|
||||||
|
|
||||||
|
- name: Get project dependencies
|
||||||
|
run: go mod download
|
||||||
|
|
||||||
|
- name: Build Transfersh
|
||||||
|
run: |
|
||||||
|
mkdir -p build_assets
|
||||||
|
go build -tags netgo -ldflags "-X github.com/dutchcoders/transfer.sh/cmd.Version=${GITHUB_REF##*/} -a -s -w -extldflags '-static'" -o build_assets/transfersh-${GITHUB_REF##*/}-${ASSET_NAME}
|
||||||
|
|
||||||
|
- name: Build Mips softfloat Transfersh
|
||||||
|
if: matrix.goarch == 'mips' || matrix.goarch == 'mipsle'
|
||||||
|
run: |
|
||||||
|
GOMIPS=softfloat go build -tags netgo -ldflags "-X github.com/dutchcoders/transfer.sh/cmd.Version=${GITHUB_REF##*/} -a -s -w -extldflags '-static'" -o build_assets/transfersh-softfloat-${GITHUB_REF##*/}-${ASSET_NAME}
|
||||||
|
|
||||||
|
- name: Rename Windows Transfersh
|
||||||
|
if: matrix.goos == 'windows'
|
||||||
|
run: |
|
||||||
|
cd ./build_assets || exit 1
|
||||||
|
mv transfersh-${GITHUB_REF##*/}-${ASSET_NAME} transfersh-${GITHUB_REF##*/}-${ASSET_NAME}.exe
|
||||||
|
|
||||||
|
- name: Prepare to release
|
||||||
|
run: |
|
||||||
|
cp ${GITHUB_WORKSPACE}/README.md ./build_assets/README.md
|
||||||
|
cp ${GITHUB_WORKSPACE}/LICENSE ./build_assets/LICENSE
|
||||||
|
|
||||||
|
- name: Create Gzip archive
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
pushd build_assets || exit 1
|
||||||
|
touch -mt $(date +%Y01010000) *
|
||||||
|
tar zcvf transfersh-${GITHUB_REF##*/}-${ASSET_NAME}.tar.gz *
|
||||||
|
mv transfersh-${GITHUB_REF##*/}-${ASSET_NAME}.tar.gz ../
|
||||||
|
FILE=`find . -name "transfersh-${GITHUB_REF##*/}-${ASSET_NAME}*"`
|
||||||
|
DGST=$FILE.sha256sum
|
||||||
|
echo `sha256sum $FILE` > $DGST
|
||||||
|
popd || exit 1
|
||||||
|
FILE=./transfersh-${GITHUB_REF##*/}-${ASSET_NAME}.tar.gz
|
||||||
|
DGST=$FILE.sha256sum
|
||||||
|
echo `sha256sum $FILE` > $DGST
|
||||||
|
|
||||||
|
- name: Change the name
|
||||||
|
run: |
|
||||||
|
mv build_assets transfersh-${GITHUB_REF##*/}-${ASSET_NAME}
|
||||||
|
|
||||||
|
- name: Upload files to Artifacts
|
||||||
|
uses: actions/upload-artifact@v2
|
||||||
|
with:
|
||||||
|
name: transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}
|
||||||
|
path: |
|
||||||
|
./transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}/*
|
||||||
|
|
||||||
|
- name: Upload binaries to release
|
||||||
|
uses: softprops/action-gh-release@v1
|
||||||
|
if: github.event_name == 'release'
|
||||||
|
with:
|
||||||
|
files: |
|
||||||
|
./transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}.tar.gz*
|
||||||
|
./transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}/transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}*
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
45
.github/workflows/test.yml
vendored
Normal file
45
.github/workflows/test.yml
vendored
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
name: test
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- "*"
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- "*"
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
go_version:
|
||||||
|
- 1.13.x
|
||||||
|
- 1.14.x
|
||||||
|
- 1.15.x
|
||||||
|
- 1.16.x
|
||||||
|
- 1.17.x
|
||||||
|
name: Test with ${{ matrix.go_version }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- uses: actions/setup-go@v1
|
||||||
|
with:
|
||||||
|
go-version: ${{ matrix.go_version }}
|
||||||
|
- name: Vet and test
|
||||||
|
run: |
|
||||||
|
go version
|
||||||
|
go vet ./...
|
||||||
|
go test ./...
|
||||||
|
golangci:
|
||||||
|
name: Linting
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- uses: actions/setup-go@v1
|
||||||
|
with:
|
||||||
|
go-version: 1.17
|
||||||
|
- name: golangci-lint
|
||||||
|
uses: golangci/golangci-lint-action@v2
|
||||||
|
with:
|
||||||
|
version: latest
|
||||||
|
skip-go-installation: true
|
||||||
|
args: "--config .golangci.yml"
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -5,6 +5,7 @@ src/
|
|||||||
bin/
|
bin/
|
||||||
*.pyc
|
*.pyc
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
|
.idea/
|
||||||
|
|
||||||
.tmp
|
.tmp
|
||||||
.vagrant
|
.vagrant
|
||||||
@ -19,3 +20,5 @@ transfersh-server/run.sh
|
|||||||
.elasticbeanstalk/*
|
.elasticbeanstalk/*
|
||||||
!.elasticbeanstalk/*.cfg.yml
|
!.elasticbeanstalk/*.cfg.yml
|
||||||
!.elasticbeanstalk/*.global.yml
|
!.elasticbeanstalk/*.global.yml
|
||||||
|
|
||||||
|
!.github/build/
|
||||||
|
|||||||
24
.golangci.yml
Normal file
24
.golangci.yml
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
run:
|
||||||
|
deadline: 10m
|
||||||
|
issues-exit-code: 1
|
||||||
|
tests: true
|
||||||
|
|
||||||
|
output:
|
||||||
|
format: colored-line-number
|
||||||
|
print-issued-lines: true
|
||||||
|
print-linter-name: true
|
||||||
|
|
||||||
|
linters:
|
||||||
|
disable:
|
||||||
|
- deadcode
|
||||||
|
- unused
|
||||||
|
|
||||||
|
issues:
|
||||||
|
max-issues-per-linter: 0
|
||||||
|
max-same-issues: 0
|
||||||
|
new: false
|
||||||
|
exclude-use-default: false
|
||||||
|
exclude-rules:
|
||||||
|
- linters:
|
||||||
|
- staticcheck
|
||||||
|
text: "SA1019:"
|
||||||
19
.travis.yml
19
.travis.yml
@ -1,19 +0,0 @@
|
|||||||
language: go
|
|
||||||
sudo: false
|
|
||||||
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- os: linux
|
|
||||||
go: 1.7.x
|
|
||||||
- os: linux
|
|
||||||
go: 1.8
|
|
||||||
- os: linux
|
|
||||||
go: tip
|
|
||||||
|
|
||||||
install:
|
|
||||||
- echo "This is an override of the default install deps step in travis."
|
|
||||||
|
|
||||||
script:
|
|
||||||
- go build -v .
|
|
||||||
- go vet $(go list ./... | grep -v vendor)
|
|
||||||
- go test $(go list ./... | grep -v vendor)
|
|
||||||
25
CODE_OF_CONDUCT.md
Normal file
25
CODE_OF_CONDUCT.md
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
|
||||||
|
# Contributor Code of Conduct
|
||||||
|
|
||||||
|
As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
|
||||||
|
|
||||||
|
We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.
|
||||||
|
|
||||||
|
Examples of unacceptable behavior by participants include:
|
||||||
|
|
||||||
|
* The use of sexualized language or imagery
|
||||||
|
* Personal attacks
|
||||||
|
* Trolling or insulting/derogatory comments
|
||||||
|
* Public or private harassment
|
||||||
|
* Publishing other's private information, such as physical or electronic addresses, without explicit permission
|
||||||
|
* Other unethical or unprofessional conduct
|
||||||
|
* Use of harsh language
|
||||||
|
|
||||||
|
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team.
|
||||||
|
|
||||||
|
This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community.
|
||||||
|
|
||||||
|
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
|
||||||
|
|
||||||
|
This Code of Conduct is adapted from the [Contributor Covenant] (https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html
|
||||||
|
|
||||||
26
Dockerfile
26
Dockerfile
@ -1,15 +1,25 @@
|
|||||||
FROM golang:1.11-alpine as build
|
# Default to Go 1.17
|
||||||
LABEL maintainer="Remco Verhoef <remco@dutchcoders.io>"
|
ARG GO_VERSION=1.17
|
||||||
|
FROM golang:${GO_VERSION}-alpine as build
|
||||||
|
|
||||||
|
# Necessary to run 'go get' and to compile the linked binary
|
||||||
|
RUN apk add git musl-dev
|
||||||
|
|
||||||
# Copy the local package files to the container's workspace.
|
|
||||||
ADD . /go/src/github.com/dutchcoders/transfer.sh
|
ADD . /go/src/github.com/dutchcoders/transfer.sh
|
||||||
|
|
||||||
# build & install server
|
WORKDIR /go/src/github.com/dutchcoders/transfer.sh
|
||||||
RUN go build -o /go/bin/transfersh github.com/dutchcoders/transfer.sh
|
|
||||||
|
|
||||||
FROM golang:1.11-alpine
|
ENV GO111MODULE=on
|
||||||
COPY --from=build /go/bin/transfersh /go/bin/transfersh
|
|
||||||
|
# build & install server
|
||||||
|
RUN CGO_ENABLED=0 go build -tags netgo -ldflags "-X github.com/dutchcoders/transfer.sh/cmd.Version=$(git describe --tags) -a -s -w -extldflags '-static'" -o /go/bin/transfersh
|
||||||
|
|
||||||
|
FROM scratch AS final
|
||||||
|
LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>"
|
||||||
|
|
||||||
|
COPY --from=build /go/bin/transfersh /go/bin/transfersh
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||||
|
|
||||||
ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"]
|
ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"]
|
||||||
|
|
||||||
EXPOSE 8080 8080
|
EXPOSE 8080
|
||||||
|
|||||||
2
LICENSE
2
LICENSE
@ -1,6 +1,8 @@
|
|||||||
The MIT License (MIT)
|
The MIT License (MIT)
|
||||||
|
|
||||||
Copyright (c) 2014-2018 DutchCoders [https://github.com/dutchcoders/]
|
Copyright (c) 2014-2018 DutchCoders [https://github.com/dutchcoders/]
|
||||||
|
Copyright (c) 2018-2020 Andrea Spacca.
|
||||||
|
Copyright (c) 2020- Andrea Spacca and Stefan Benten.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|||||||
5
Makefile
Normal file
5
Makefile
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
.PHONY: lint
|
||||||
|
|
||||||
|
lint:
|
||||||
|
golangci-lint run --out-format=github-actions --config .golangci.yml
|
||||||
|
|
||||||
231
README.md
231
README.md
@ -1,8 +1,12 @@
|
|||||||
# transfer.sh [](https://gitter.im/dutchcoders/transfer.sh?utm_source=badge&utm_medium=badge&utm_campaign=&utm_campaign=pr-badge&utm_content=badge) [](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [](https://hub.docker.com/r/dutchcoders/transfer.sh/) [](https://travis-ci.org/dutchcoders/transfer.sh)
|
# transfer.sh [](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [](https://hub.docker.com/r/dutchcoders/transfer.sh/) [](https://github.com/dutchcoders/transfer.sh/actions/workflows/test.yml?query=branch%3Amain)
|
||||||
|
|
||||||
Easy and fast file sharing from the command-line. This code contains the server with everything you need to create your own instance.
|
Easy and fast file sharing from the command-line. This code contains the server with everything you need to create your own instance.
|
||||||
|
|
||||||
Transfer.sh currently supports the s3 (Amazon S3), gdrive (Google Drive) providers, and local file system (local).
|
Transfer.sh currently supports the s3 (Amazon S3), gdrive (Google Drive), storj (Storj) providers, and local file system (local).
|
||||||
|
|
||||||
|
## Disclaimer
|
||||||
|
|
||||||
|
The service at transfersh.com is of unknown origin and reported as cloud malware.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@ -11,90 +15,62 @@ Transfer.sh currently supports the s3 (Amazon S3), gdrive (Google Drive) provide
|
|||||||
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt
|
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
### Encrypt & upload:
|
### Encrypt & Upload:
|
||||||
```bash
|
```bash
|
||||||
$ cat /tmp/hello.txt|gpg -ac -o-|curl -X PUT --upload-file "-" https://transfer.sh/test.txt
|
$ cat /tmp/hello.txt|gpg -ac -o-|curl -X PUT --upload-file "-" https://transfer.sh/test.txt
|
||||||
````
|
````
|
||||||
|
|
||||||
### Download & decrypt:
|
### Download & Decrypt:
|
||||||
```bash
|
```bash
|
||||||
$ curl https://transfer.sh/1lDau/test.txt|gpg -o- > /tmp/hello.txt
|
$ curl https://transfer.sh/1lDau/test.txt|gpg -o- > /tmp/hello.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
### Upload to virustotal:
|
### Upload to Virustotal:
|
||||||
```bash
|
```bash
|
||||||
$ curl -X PUT --upload-file nhgbhhj https://transfer.sh/test.txt/virustotal
|
$ curl -X PUT --upload-file nhgbhhj https://transfer.sh/test.txt/virustotal
|
||||||
```
|
```
|
||||||
|
|
||||||
## Add alias to .bashrc or .zshrc
|
### Deleting
|
||||||
|
|
||||||
### Using curl
|
|
||||||
```bash
|
```bash
|
||||||
transfer() {
|
$ curl -X DELETE <X-Url-Delete Response Header URL>
|
||||||
curl --progress-bar --upload-file "$1" https://transfer.sh/$(basename $1) | tee /dev/null;
|
|
||||||
}
|
|
||||||
|
|
||||||
alias transfer=transfer
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Using wget
|
## Request Headers
|
||||||
|
|
||||||
|
### Max-Downloads
|
||||||
```bash
|
```bash
|
||||||
transfer() {
|
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "Max-Downloads: 1" # Limit the number of downloads
|
||||||
wget -t 1 -qO - --method=PUT --body-file="$1" --header="Content-Type: $(file -b --mime-type $1)" https://transfer.sh/$(basename $1);
|
|
||||||
}
|
|
||||||
|
|
||||||
alias transfer=transfer
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Add alias for fish-shell
|
### Max-Days
|
||||||
|
|
||||||
### Using curl
|
|
||||||
```bash
|
```bash
|
||||||
function transfer --description 'Upload a file to transfer.sh'
|
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "Max-Days: 1" # Set the number of days before deletion
|
||||||
if [ $argv[1] ]
|
|
||||||
# write to output to tmpfile because of progress bar
|
|
||||||
set -l tmpfile ( mktemp -t transferXXX )
|
|
||||||
curl --progress-bar --upload-file "$argv[1]" https://transfer.sh/(basename $argv[1]) >> $tmpfile
|
|
||||||
cat $tmpfile
|
|
||||||
command rm -f $tmpfile
|
|
||||||
else
|
|
||||||
echo 'usage: transfer FILE_TO_TRANSFER'
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
funcsave transfer
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Using wget
|
### X-Encrypt-Password
|
||||||
```bash
|
```bash
|
||||||
function transfer --description 'Upload a file to transfer.sh'
|
$ curl --upload-file ./hello.txt https://transfer.sh/hello.txt -H "X-Encrypt-Password: test" # Encrypt the content sever side with AES265 using "test" as password
|
||||||
if [ $argv[1] ]
|
|
||||||
wget -t 1 -qO - --method=PUT --body-file="$argv[1]" --header="Content-Type: (file -b --mime-type $argv[1])" https://transfer.sh/(basename $argv[1])
|
|
||||||
else
|
|
||||||
echo 'usage: transfer FILE_TO_TRANSFER'
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
funcsave transfer
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Now run it like this:
|
### X-Decrypt-Password
|
||||||
```bash
|
```bash
|
||||||
$ transfer test.txt
|
$ curl https://transfer.sh/BAYh0/hello.txt -H "X-Decrypt-Password: test" # Decrypt the content sever side with AES265 using "test" as password
|
||||||
```
|
```
|
||||||
|
|
||||||
## Add alias on Windows
|
## Response Headers
|
||||||
|
|
||||||
Put a file called `transfer.cmd` somewhere in your PATH with this inside it:
|
### X-Url-Delete
|
||||||
```cmd
|
|
||||||
@echo off
|
The URL used to request the deletion of a file and returned as a response header.
|
||||||
setlocal
|
```bash
|
||||||
:: use env vars to pass names to PS, to avoid escaping issues
|
curl -sD - --upload-file ./hello https://transfer.sh/hello.txt | grep 'X-Url-Delete'
|
||||||
set FN=%~nx1
|
X-Url-Delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
|
||||||
set FULL=%1
|
|
||||||
powershell -noprofile -command "$(Invoke-Webrequest -Method put -Infile $Env:FULL https://transfer.sh/$Env:FN).Content"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
See good usage examples on [examples.md](examples.md)
|
||||||
|
|
||||||
## Link aliases
|
## Link aliases
|
||||||
|
|
||||||
Create direct download link:
|
Create direct download link:
|
||||||
@ -109,45 +85,67 @@ https://transfer.sh/1lDau/test.txt --> https://transfer.sh/inline/1lDau/test.txt
|
|||||||
|
|
||||||
Parameter | Description | Value | Env
|
Parameter | Description | Value | Env
|
||||||
--- | --- | --- | ---
|
--- | --- | --- | ---
|
||||||
listener | port to use for http (:80) | |
|
listener | port to use for http (:80) | | LISTENER |
|
||||||
profile-listener | port to use for profiler (:6060)| |
|
profile-listener | port to use for profiler (:6060) | | PROFILE_LISTENER |
|
||||||
force-https | redirect to https | false |
|
force-https | redirect to https | false | FORCE_HTTPS
|
||||||
tls-listener | port to use for https (:443) | |
|
tls-listener | port to use for https (:443) | | TLS_LISTENER |
|
||||||
tls-listener-only | flag to enable tls listener only | |
|
tls-listener-only | flag to enable tls listener only | | TLS_LISTENER_ONLY |
|
||||||
tls-cert-file | path to tls certificate | |
|
tls-cert-file | path to tls certificate | | TLS_CERT_FILE |
|
||||||
tls-private-key | path to tls private key | |
|
tls-private-key | path to tls private key | | TLS_PRIVATE_KEY |
|
||||||
http-auth-user | user for basic http auth on upload | |
|
http-auth-user | user for basic http auth on upload | | HTTP_AUTH_USER |
|
||||||
http-auth-pass | pass for basic http auth on upload | |
|
http-auth-pass | pass for basic http auth on upload | | HTTP_AUTH_PASS |
|
||||||
temp-path | path to temp folder | system temp |
|
ip-whitelist | comma separated list of ips allowed to connect to the service | | IP_WHITELIST |
|
||||||
web-path | path to static web files (for development) | |
|
ip-blacklist | comma separated list of ips not allowed to connect to the service | | IP_BLACKLIST |
|
||||||
ga-key | google analytics key for the front end | |
|
temp-path | path to temp folder | system temp | TEMP_PATH |
|
||||||
uservoice-key | user voice key for the front end | |
|
web-path | path to static web files (for development or custom front end) | | WEB_PATH |
|
||||||
provider | which storage provider to use | (s3, grdrive or local) |
|
proxy-path | path prefix when service is run behind a proxy | | PROXY_PATH |
|
||||||
aws-access-key | aws access key | | AWS_ACCESS_KEY
|
proxy-port | port of the proxy when the service is run behind a proxy | | PROXY_PORT |
|
||||||
aws-secret-key | aws access key | | AWS_SECRET_KEY
|
email-contact | email contact for the front end | | EMAIL_CONTACT |
|
||||||
bucket | aws bucket | | BUCKET
|
ga-key | google analytics key for the front end | | GA_KEY |
|
||||||
basedir | path storage for local/gdrive provider| |
|
provider | which storage provider to use | (s3, storj, gdrive or local) |
|
||||||
gdrive-client-json-filepath | path to oauth client json config for gdrive provider| |
|
uservoice-key | user voice key for the front end | | USERVOICE_KEY |
|
||||||
gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| |
|
aws-access-key | aws access key | | AWS_ACCESS_KEY |
|
||||||
lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated) | |
|
aws-secret-key | aws access key | | AWS_SECRET_KEY |
|
||||||
log | path to log file| |
|
bucket | aws bucket | | BUCKET |
|
||||||
|
s3-endpoint | Custom S3 endpoint. | | S3_ENDPOINT |
|
||||||
|
s3-region | region of the s3 bucket | eu-west-1 | S3_REGION |
|
||||||
|
s3-no-multipart | disables s3 multipart upload | false | S3_NO_MULTIPART |
|
||||||
|
s3-path-style | Forces path style URLs, required for Minio. | false | S3_PATH_STYLE |
|
||||||
|
storj-access | Access for the project | | STORJ_ACCESS |
|
||||||
|
storj-bucket | Bucket to use within the project | | STORJ_BUCKET |
|
||||||
|
basedir | path storage for local/gdrive provider | | BASEDIR |
|
||||||
|
gdrive-client-json-filepath | path to oauth client json config for gdrive provider | | GDRIVE_CLIENT_JSON_FILEPATH |
|
||||||
|
gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| | GDRIVE_LOCAL_CONFIG_PATH |
|
||||||
|
gdrive-chunk-size | chunk size for gdrive upload in megabytes, must be lower than available memory (8 MB) | | GDRIVE_CHUNK_SIZE |
|
||||||
|
lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated) | | HOSTS |
|
||||||
|
log | path to log file| | LOG |
|
||||||
|
cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS |
|
||||||
|
clamav-host | host for clamav feature | | CLAMAV_HOST |
|
||||||
|
perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |
|
||||||
|
rate-limit | request per minute | | RATE_LIMIT |
|
||||||
|
max-upload-size | max upload size in kilobytes | | MAX_UPLOAD_SIZE |
|
||||||
|
purge-days | number of days after the uploads are purged automatically | | PURGE_DAYS |
|
||||||
|
purge-interval | interval in hours to run the automatic purge for (not applicable to S3 and Storj) | | PURGE_INTERVAL |
|
||||||
|
random-token-length | length of the random token for the upload path (double the size for delete path) | 6 | RANDOM_TOKEN_LENGTH |
|
||||||
|
|
||||||
If you want to use TLS using lets encrypt certificates, set lets-encrypt-hosts to your domain, set tls-listener to :443 and enable force-https.
|
If you want to use TLS using lets encrypt certificates, set lets-encrypt-hosts to your domain, set tls-listener to :443 and enable force-https.
|
||||||
|
|
||||||
If you want to use TLS using your own certificates, set tls-listener to :443, force-https, tls-cert=file and tls-private-key.
|
If you want to use TLS using your own certificates, set tls-listener to :443, force-https, tls-cert-file and tls-private-key.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
Make sure your GOPATH is set correctly.
|
Switched to GO111MODULE
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go run main.go -provider=local --listener :8080 --temp-path=/tmp/ --basedir=/tmp/
|
go run main.go --provider=local --listener :8080 --temp-path=/tmp/ --basedir=/tmp/
|
||||||
```
|
```
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go build -o transfersh main.go
|
$ git clone git@github.com:dutchcoders/transfer.sh.git
|
||||||
|
$ cd transfer.sh
|
||||||
|
$ go build -o transfersh main.go
|
||||||
```
|
```
|
||||||
|
|
||||||
## Docker
|
## Docker
|
||||||
@ -158,6 +156,64 @@ For easy deployment, we've created a Docker container.
|
|||||||
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/
|
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## S3 Usage
|
||||||
|
|
||||||
|
For the usage with a AWS S3 Bucket, you just need to specify the following options:
|
||||||
|
- provider
|
||||||
|
- aws-access-key
|
||||||
|
- aws-secret-key
|
||||||
|
- bucket
|
||||||
|
- s3-region
|
||||||
|
|
||||||
|
If you specify the s3-region, you don't need to set the endpoint URL since the correct endpoint will used automatically.
|
||||||
|
|
||||||
|
### Custom S3 providers
|
||||||
|
|
||||||
|
To use a custom non-AWS S3 provider, you need to specify the endpoint as defined from your cloud provider.
|
||||||
|
|
||||||
|
## Storj Network Provider
|
||||||
|
|
||||||
|
To use the Storj Network as a storage provider you need to specify the following flags:
|
||||||
|
- provider `--provider storj`
|
||||||
|
- storj-access _(either via flag or environment variable STORJ_ACCESS)_
|
||||||
|
- storj-bucket _(either via flag or environment variable STORJ_BUCKET)_
|
||||||
|
|
||||||
|
### Creating Bucket and Scope
|
||||||
|
|
||||||
|
You need to create an access grant (or copy it from the uplink configuration) and a bucket in preparation.
|
||||||
|
|
||||||
|
To get started, log in to your account and go to the Access Grant Menu and start the Wizard on the upper right.
|
||||||
|
|
||||||
|
Enter your access grant name of choice, hit *Next* and restrict it as necessary/preferred.
|
||||||
|
Afterwards continue either in CLI or within the Browser. Next, you'll be asked for a Passphrase used as Encryption Key.
|
||||||
|
**Make sure to save it in a safe place. Without it, you will lose the ability to decrypt your files!**
|
||||||
|
|
||||||
|
Afterwards, you can copy the access grant and then start the startup of the transfer.sh endpoint.
|
||||||
|
It is recommended to provide both the access grant and the bucket name as ENV Variables for enhanced security.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
```
|
||||||
|
export STORJ_BUCKET=<BUCKET NAME>
|
||||||
|
export STORJ_ACCESS=<ACCESS GRANT>
|
||||||
|
transfer.sh --provider storj
|
||||||
|
```
|
||||||
|
|
||||||
|
## Google Drive Usage
|
||||||
|
|
||||||
|
For the usage with Google drive, you need to specify the following options:
|
||||||
|
- provider
|
||||||
|
- gdrive-client-json-filepath
|
||||||
|
- gdrive-local-config-path
|
||||||
|
- basedir
|
||||||
|
|
||||||
|
### Creating Gdrive Client Json
|
||||||
|
|
||||||
|
You need to create an OAuth Client id from console.cloud.google.com, download the file, and place it into a safe directory.
|
||||||
|
|
||||||
|
### Usage example
|
||||||
|
|
||||||
|
```go run main.go --provider gdrive --basedir /tmp/ --gdrive-client-json-filepath /[credential_dir] --gdrive-local-config-path [directory_to_save_config] ```
|
||||||
|
|
||||||
## Contributions
|
## Contributions
|
||||||
|
|
||||||
Contributions are welcome.
|
Contributions are welcome.
|
||||||
@ -170,7 +226,16 @@ Contributions are welcome.
|
|||||||
|
|
||||||
**Uvis Grinfelds**
|
**Uvis Grinfelds**
|
||||||
|
|
||||||
## Copyright and license
|
## Maintainer
|
||||||
|
|
||||||
|
**Andrea Spacca**
|
||||||
|
|
||||||
|
**Stefan Benten**
|
||||||
|
|
||||||
|
## Copyright and License
|
||||||
|
|
||||||
Code and documentation copyright 2011-2018 Remco Verhoef.
|
Code and documentation copyright 2011-2018 Remco Verhoef.
|
||||||
|
Code and documentation copyright 2018-2020 Andrea Spacca.
|
||||||
|
Code and documentation copyright 2020- Andrea Spacca and Stefan Benten.
|
||||||
|
|
||||||
Code released under [the MIT license](LICENSE).
|
Code released under [the MIT license](LICENSE).
|
||||||
|
|||||||
334
cmd/cmd.go
334
cmd/cmd.go
@ -2,17 +2,18 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/dutchcoders/transfer.sh/server"
|
"github.com/dutchcoders/transfer.sh/server"
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
"github.com/minio/cli"
|
"github.com/urfave/cli"
|
||||||
|
"google.golang.org/api/googleapi"
|
||||||
)
|
)
|
||||||
|
|
||||||
var Version = "0.1"
|
// Version is inject at build time
|
||||||
|
var Version = "0.0.0"
|
||||||
var helpTemplate = `NAME:
|
var helpTemplate = `NAME:
|
||||||
{{.Name}} - {{.Usage}}
|
{{.Name}} - {{.Usage}}
|
||||||
|
|
||||||
@ -34,69 +35,105 @@ VERSION:
|
|||||||
|
|
||||||
var globalFlags = []cli.Flag{
|
var globalFlags = []cli.Flag{
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "listener",
|
Name: "listener",
|
||||||
Usage: "127.0.0.1:8080",
|
Usage: "127.0.0.1:8080",
|
||||||
Value: "127.0.0.1:8080",
|
Value: "127.0.0.1:8080",
|
||||||
|
EnvVar: "LISTENER",
|
||||||
},
|
},
|
||||||
// redirect to https?
|
// redirect to https?
|
||||||
// hostnames
|
// hostnames
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "profile-listener",
|
Name: "profile-listener",
|
||||||
Usage: "127.0.0.1:6060",
|
Usage: "127.0.0.1:6060",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "PROFILE_LISTENER",
|
||||||
},
|
},
|
||||||
cli.BoolFlag{
|
cli.BoolFlag{
|
||||||
Name: "force-https",
|
Name: "force-https",
|
||||||
Usage: "",
|
Usage: "",
|
||||||
|
EnvVar: "FORCE_HTTPS",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "tls-listener",
|
Name: "tls-listener",
|
||||||
Usage: "127.0.0.1:8443",
|
Usage: "127.0.0.1:8443",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "TLS_LISTENER",
|
||||||
},
|
},
|
||||||
cli.BoolFlag{
|
cli.BoolFlag{
|
||||||
Name: "tls-listener-only",
|
Name: "tls-listener-only",
|
||||||
Usage: "",
|
Usage: "",
|
||||||
|
EnvVar: "TLS_LISTENER_ONLY",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "tls-cert-file",
|
Name: "tls-cert-file",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "TLS_CERT_FILE",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "tls-private-key",
|
Name: "tls-private-key",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "TLS_PRIVATE_KEY",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "temp-path",
|
Name: "temp-path",
|
||||||
Usage: "path to temp files",
|
Usage: "path to temp files",
|
||||||
Value: os.TempDir(),
|
Value: os.TempDir(),
|
||||||
|
EnvVar: "TEMP_PATH",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "web-path",
|
Name: "web-path",
|
||||||
Usage: "path to static web files",
|
Usage: "path to static web files",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "WEB_PATH",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "ga-key",
|
Name: "proxy-path",
|
||||||
Usage: "key for google analytics (front end)",
|
Usage: "path prefix when service is run behind a proxy",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "PROXY_PATH",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "uservoice-key",
|
Name: "proxy-port",
|
||||||
Usage: "key for user voice (front end)",
|
Usage: "port of the proxy when the service is run behind a proxy",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "PROXY_PORT",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "provider",
|
Name: "email-contact",
|
||||||
Usage: "s3|gdrive|local",
|
Usage: "email address to link in Contact Us (front end)",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "EMAIL_CONTACT",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "ga-key",
|
||||||
|
Usage: "key for google analytics (front end)",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "GA_KEY",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "uservoice-key",
|
||||||
|
Usage: "key for user voice (front end)",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "USERVOICE_KEY",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "provider",
|
||||||
|
Usage: "s3|gdrive|local",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "PROVIDER",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "s3-endpoint",
|
Name: "s3-endpoint",
|
||||||
Usage: "",
|
Usage: "",
|
||||||
Value: "http://s3-eu-west-1.amazonaws.com",
|
Value: "",
|
||||||
EnvVar: "S3_ENDPOINT",
|
EnvVar: "S3_ENDPOINT",
|
||||||
},
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "s3-region",
|
||||||
|
Usage: "",
|
||||||
|
Value: "eu-west-1",
|
||||||
|
EnvVar: "S3_REGION",
|
||||||
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "aws-access-key",
|
Name: "aws-access-key",
|
||||||
Usage: "",
|
Usage: "",
|
||||||
@ -115,21 +152,69 @@ var globalFlags = []cli.Flag{
|
|||||||
Value: "",
|
Value: "",
|
||||||
EnvVar: "BUCKET",
|
EnvVar: "BUCKET",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.BoolFlag{
|
||||||
Name: "gdrive-client-json-filepath",
|
Name: "s3-no-multipart",
|
||||||
Usage: "",
|
Usage: "Disables S3 Multipart Puts",
|
||||||
Value: "",
|
EnvVar: "S3_NO_MULTIPART",
|
||||||
|
},
|
||||||
|
cli.BoolFlag{
|
||||||
|
Name: "s3-path-style",
|
||||||
|
Usage: "Forces path style URLs, required for Minio.",
|
||||||
|
EnvVar: "S3_PATH_STYLE",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "gdrive-local-config-path",
|
Name: "gdrive-client-json-filepath",
|
||||||
Usage: "",
|
Usage: "",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "GDRIVE_CLIENT_JSON_FILEPATH",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "gdrive-local-config-path",
|
||||||
|
Usage: "",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "GDRIVE_LOCAL_CONFIG_PATH",
|
||||||
|
},
|
||||||
|
cli.IntFlag{
|
||||||
|
Name: "gdrive-chunk-size",
|
||||||
|
Usage: "",
|
||||||
|
Value: googleapi.DefaultUploadChunkSize / 1024 / 1024,
|
||||||
|
EnvVar: "GDRIVE_CHUNK_SIZE",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "storj-access",
|
||||||
|
Usage: "Access for the project",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "STORJ_ACCESS",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "storj-bucket",
|
||||||
|
Usage: "Bucket to use within the project",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "STORJ_BUCKET",
|
||||||
},
|
},
|
||||||
cli.IntFlag{
|
cli.IntFlag{
|
||||||
Name: "rate-limit",
|
Name: "rate-limit",
|
||||||
Usage: "requests per minute",
|
Usage: "requests per minute",
|
||||||
Value: 0,
|
Value: 0,
|
||||||
EnvVar: "",
|
EnvVar: "RATE_LIMIT",
|
||||||
|
},
|
||||||
|
cli.IntFlag{
|
||||||
|
Name: "purge-days",
|
||||||
|
Usage: "number of days after uploads are purged automatically",
|
||||||
|
Value: 0,
|
||||||
|
EnvVar: "PURGE_DAYS",
|
||||||
|
},
|
||||||
|
cli.IntFlag{
|
||||||
|
Name: "purge-interval",
|
||||||
|
Usage: "interval in hours to run the automatic purge for",
|
||||||
|
Value: 0,
|
||||||
|
EnvVar: "PURGE_INTERVAL",
|
||||||
|
},
|
||||||
|
cli.Int64Flag{
|
||||||
|
Name: "max-upload-size",
|
||||||
|
Usage: "max limit for upload, in kilobytes",
|
||||||
|
Value: 0,
|
||||||
|
EnvVar: "MAX_UPLOAD_SIZE",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "lets-encrypt-hosts",
|
Name: "lets-encrypt-hosts",
|
||||||
@ -138,14 +223,16 @@ var globalFlags = []cli.Flag{
|
|||||||
EnvVar: "HOSTS",
|
EnvVar: "HOSTS",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "log",
|
Name: "log",
|
||||||
Usage: "/var/log/transfersh.log",
|
Usage: "/var/log/transfersh.log",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "LOG",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "basedir",
|
Name: "basedir",
|
||||||
Usage: "path to storage",
|
Usage: "path to storage",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "BASEDIR",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "clamav-host",
|
Name: "clamav-host",
|
||||||
@ -153,6 +240,11 @@ var globalFlags = []cli.Flag{
|
|||||||
Value: "",
|
Value: "",
|
||||||
EnvVar: "CLAMAV_HOST",
|
EnvVar: "CLAMAV_HOST",
|
||||||
},
|
},
|
||||||
|
cli.BoolFlag{
|
||||||
|
Name: "perform-clamav-prescan",
|
||||||
|
Usage: "perform-clamav-prescan",
|
||||||
|
EnvVar: "PERFORM_CLAMAV_PRESCAN",
|
||||||
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "virustotal-key",
|
Name: "virustotal-key",
|
||||||
Usage: "virustotal-key",
|
Usage: "virustotal-key",
|
||||||
@ -160,41 +252,73 @@ var globalFlags = []cli.Flag{
|
|||||||
EnvVar: "VIRUSTOTAL_KEY",
|
EnvVar: "VIRUSTOTAL_KEY",
|
||||||
},
|
},
|
||||||
cli.BoolFlag{
|
cli.BoolFlag{
|
||||||
Name: "profiler",
|
Name: "profiler",
|
||||||
Usage: "enable profiling",
|
Usage: "enable profiling",
|
||||||
|
EnvVar: "PROFILER",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "http-auth-user",
|
Name: "http-auth-user",
|
||||||
Usage: "user for http basic auth",
|
Usage: "user for http basic auth",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "HTTP_AUTH_USER",
|
||||||
},
|
},
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "http-auth-pass",
|
Name: "http-auth-pass",
|
||||||
Usage: "pass for http basic auth",
|
Usage: "pass for http basic auth",
|
||||||
Value: "",
|
Value: "",
|
||||||
|
EnvVar: "HTTP_AUTH_PASS",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "ip-whitelist",
|
||||||
|
Usage: "comma separated list of ips allowed to connect to the service",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "IP_WHITELIST",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "ip-blacklist",
|
||||||
|
Usage: "comma separated list of ips not allowed to connect to the service",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "IP_BLACKLIST",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "cors-domains",
|
||||||
|
Usage: "comma separated list of domains allowed for CORS requests",
|
||||||
|
Value: "",
|
||||||
|
EnvVar: "CORS_DOMAINS",
|
||||||
|
},
|
||||||
|
cli.IntFlag{
|
||||||
|
Name: "random-token-length",
|
||||||
|
Usage: "",
|
||||||
|
Value: 6,
|
||||||
|
EnvVar: "RANDOM_TOKEN_LENGTH",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cmd wraps cli.app
|
||||||
type Cmd struct {
|
type Cmd struct {
|
||||||
*cli.App
|
*cli.App
|
||||||
}
|
}
|
||||||
|
|
||||||
func VersionAction(c *cli.Context) {
|
func versionAction(c *cli.Context) {
|
||||||
fmt.Println(color.YellowString(fmt.Sprintf("transfer.sh: Easy file sharing from the command line")))
|
fmt.Println(color.YellowString(fmt.Sprintf("transfer.sh %s: Easy file sharing from the command line", Version)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// New is the factory for transfer.sh
|
||||||
func New() *Cmd {
|
func New() *Cmd {
|
||||||
|
logger := log.New(os.Stdout, "[transfer.sh]", log.LstdFlags)
|
||||||
|
|
||||||
app := cli.NewApp()
|
app := cli.NewApp()
|
||||||
app.Name = "transfer.sh"
|
app.Name = "transfer.sh"
|
||||||
app.Author = ""
|
app.Author = ""
|
||||||
app.Usage = "transfer.sh"
|
app.Usage = "transfer.sh"
|
||||||
app.Description = `Easy file sharing from the command line`
|
app.Description = `Easy file sharing from the command line`
|
||||||
|
app.Version = Version
|
||||||
app.Flags = globalFlags
|
app.Flags = globalFlags
|
||||||
app.CustomAppHelpTemplate = helpTemplate
|
app.CustomAppHelpTemplate = helpTemplate
|
||||||
app.Commands = []cli.Command{
|
app.Commands = []cli.Command{
|
||||||
{
|
{
|
||||||
Name: "version",
|
Name: "version",
|
||||||
Action: VersionAction,
|
Action: versionAction,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -208,6 +332,10 @@ func New() *Cmd {
|
|||||||
options = append(options, server.Listener(v))
|
options = append(options, server.Listener(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v := c.String("cors-domains"); v != "" {
|
||||||
|
options = append(options, server.CorsDomains(v))
|
||||||
|
}
|
||||||
|
|
||||||
if v := c.String("tls-listener"); v == "" {
|
if v := c.String("tls-listener"); v == "" {
|
||||||
} else if c.Bool("tls-listener-only") {
|
} else if c.Bool("tls-listener-only") {
|
||||||
options = append(options, server.TLSListener(v, true))
|
options = append(options, server.TLSListener(v, true))
|
||||||
@ -223,6 +351,18 @@ func New() *Cmd {
|
|||||||
options = append(options, server.WebPath(v))
|
options = append(options, server.WebPath(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v := c.String("proxy-path"); v != "" {
|
||||||
|
options = append(options, server.ProxyPath(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := c.String("proxy-port"); v != "" {
|
||||||
|
options = append(options, server.ProxyPort(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := c.String("email-contact"); v != "" {
|
||||||
|
options = append(options, server.EmailContact(v))
|
||||||
|
}
|
||||||
|
|
||||||
if v := c.String("ga-key"); v != "" {
|
if v := c.String("ga-key"); v != "" {
|
||||||
options = append(options, server.GoogleAnalytics(v))
|
options = append(options, server.GoogleAnalytics(v))
|
||||||
}
|
}
|
||||||
@ -235,6 +375,12 @@ func New() *Cmd {
|
|||||||
options = append(options, server.TempPath(v))
|
options = append(options, server.TempPath(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v := c.String("log"); v != "" {
|
||||||
|
options = append(options, server.LogFile(logger, v))
|
||||||
|
} else {
|
||||||
|
options = append(options, server.Logger(logger))
|
||||||
|
}
|
||||||
|
|
||||||
if v := c.String("lets-encrypt-hosts"); v != "" {
|
if v := c.String("lets-encrypt-hosts"); v != "" {
|
||||||
options = append(options, server.UseLetsEncrypt(strings.Split(v, ",")))
|
options = append(options, server.UseLetsEncrypt(strings.Split(v, ",")))
|
||||||
}
|
}
|
||||||
@ -247,10 +393,31 @@ func New() *Cmd {
|
|||||||
options = append(options, server.ClamavHost(v))
|
options = append(options, server.ClamavHost(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v := c.Bool("perform-clamav-prescan"); v {
|
||||||
|
if c.String("clamav-host") == "" {
|
||||||
|
panic("clamav-host not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
options = append(options, server.PerformClamavPrescan(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := c.Int64("max-upload-size"); v > 0 {
|
||||||
|
options = append(options, server.MaxUploadSize(v))
|
||||||
|
}
|
||||||
|
|
||||||
if v := c.Int("rate-limit"); v > 0 {
|
if v := c.Int("rate-limit"); v > 0 {
|
||||||
options = append(options, server.RateLimit(v))
|
options = append(options, server.RateLimit(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
v := c.Int("random-token-length")
|
||||||
|
options = append(options, server.RandomTokenLength(v))
|
||||||
|
|
||||||
|
purgeDays := c.Int("purge-days")
|
||||||
|
purgeInterval := c.Int("purge-interval")
|
||||||
|
if purgeDays > 0 && purgeInterval > 0 {
|
||||||
|
options = append(options, server.Purge(purgeDays, purgeInterval))
|
||||||
|
}
|
||||||
|
|
||||||
if cert := c.String("tls-cert-file"); cert == "" {
|
if cert := c.String("tls-cert-file"); cert == "" {
|
||||||
} else if pk := c.String("tls-private-key"); pk == "" {
|
} else if pk := c.String("tls-private-key"); pk == "" {
|
||||||
} else {
|
} else {
|
||||||
@ -262,13 +429,30 @@ func New() *Cmd {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if c.Bool("force-https") {
|
if c.Bool("force-https") {
|
||||||
options = append(options, server.ForceHTTPs())
|
options = append(options, server.ForceHTTPS())
|
||||||
}
|
}
|
||||||
|
|
||||||
if httpAuthUser := c.String("http-auth-user"); httpAuthUser == "" {
|
if httpAuthUser := c.String("http-auth-user"); httpAuthUser == "" {
|
||||||
} else if httpAuthPass := c.String("http-auth-pass"); httpAuthPass == "" {
|
} else if httpAuthPass := c.String("http-auth-pass"); httpAuthPass == "" {
|
||||||
} else {
|
} else {
|
||||||
options = append(options, server.HttpAuthCredentials(httpAuthUser, httpAuthPass))
|
options = append(options, server.HTTPAuthCredentials(httpAuthUser, httpAuthPass))
|
||||||
|
}
|
||||||
|
|
||||||
|
applyIPFilter := false
|
||||||
|
ipFilterOptions := server.IPFilterOptions{}
|
||||||
|
if ipWhitelist := c.String("ip-whitelist"); ipWhitelist != "" {
|
||||||
|
applyIPFilter = true
|
||||||
|
ipFilterOptions.AllowedIPs = strings.Split(ipWhitelist, ",")
|
||||||
|
ipFilterOptions.BlockByDefault = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if ipBlacklist := c.String("ip-blacklist"); ipBlacklist != "" {
|
||||||
|
applyIPFilter = true
|
||||||
|
ipFilterOptions.BlockedIPs = strings.Split(ipBlacklist, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
if applyIPFilter {
|
||||||
|
options = append(options, server.FilterOptions(ipFilterOptions))
|
||||||
}
|
}
|
||||||
|
|
||||||
switch provider := c.String("provider"); provider {
|
switch provider := c.String("provider"); provider {
|
||||||
@ -279,19 +463,31 @@ func New() *Cmd {
|
|||||||
panic("secret-key not set.")
|
panic("secret-key not set.")
|
||||||
} else if bucket := c.String("bucket"); bucket == "" {
|
} else if bucket := c.String("bucket"); bucket == "" {
|
||||||
panic("bucket not set.")
|
panic("bucket not set.")
|
||||||
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint")); err != nil {
|
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
} else {
|
} else {
|
||||||
options = append(options, server.UseStorage(storage))
|
options = append(options, server.UseStorage(storage))
|
||||||
}
|
}
|
||||||
case "gdrive":
|
case "gdrive":
|
||||||
if clientJsonFilepath := c.String("gdrive-client-json-filepath"); clientJsonFilepath == "" {
|
chunkSize := c.Int("gdrive-chunk-size")
|
||||||
|
|
||||||
|
if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
|
||||||
panic("client-json-filepath not set.")
|
panic("client-json-filepath not set.")
|
||||||
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
|
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
|
||||||
panic("local-config-path not set.")
|
panic("local-config-path not set.")
|
||||||
} else if basedir := c.String("basedir"); basedir == "" {
|
} else if basedir := c.String("basedir"); basedir == "" {
|
||||||
panic("basedir not set.")
|
panic("basedir not set.")
|
||||||
} else if storage, err := server.NewGDriveStorage(clientJsonFilepath, localConfigPath, basedir); err != nil {
|
} else if storage, err := server.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
|
||||||
|
panic(err)
|
||||||
|
} else {
|
||||||
|
options = append(options, server.UseStorage(storage))
|
||||||
|
}
|
||||||
|
case "storj":
|
||||||
|
if access := c.String("storj-access"); access == "" {
|
||||||
|
panic("storj-access not set.")
|
||||||
|
} else if bucket := c.String("storj-bucket"); bucket == "" {
|
||||||
|
panic("storj-bucket not set.")
|
||||||
|
} else if storage, err := server.NewStorjStorage(access, bucket, purgeDays, logger); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
} else {
|
} else {
|
||||||
options = append(options, server.UseStorage(storage))
|
options = append(options, server.UseStorage(storage))
|
||||||
@ -299,7 +495,7 @@ func New() *Cmd {
|
|||||||
case "local":
|
case "local":
|
||||||
if v := c.String("basedir"); v == "" {
|
if v := c.String("basedir"); v == "" {
|
||||||
panic("basedir not set.")
|
panic("basedir not set.")
|
||||||
} else if storage, err := server.NewLocalStorage(v); err != nil {
|
} else if storage, err := server.NewLocalStorage(v, logger); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
} else {
|
} else {
|
||||||
options = append(options, server.UseStorage(storage))
|
options = append(options, server.UseStorage(storage))
|
||||||
@ -313,7 +509,7 @@ func New() *Cmd {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(color.RedString("Error starting server: %s", err.Error()))
|
logger.Println(color.RedString("Error starting server: %s", err.Error()))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
263
examples.md
Normal file
263
examples.md
Normal file
@ -0,0 +1,263 @@
|
|||||||
|
# Table of Contents
|
||||||
|
|
||||||
|
* [Aliases](#aliases)
|
||||||
|
* [Uploading and downloading](#uploading-and-downloading)
|
||||||
|
* [Archiving and backups](#archiving-and-backups)
|
||||||
|
* [Encrypting and decrypting](#encrypting-and-decrypting)
|
||||||
|
* [Scanning for viruses](#scanning-for-viruses)
|
||||||
|
* [Uploading and copy download command](#uploading-and-copy-download-command)
|
||||||
|
|
||||||
|
## Aliases
|
||||||
|
<a name="aliases"/>
|
||||||
|
|
||||||
|
## Add alias to .bashrc or .zshrc
|
||||||
|
|
||||||
|
### Using curl
|
||||||
|
```bash
|
||||||
|
transfer() {
|
||||||
|
curl --progress-bar --upload-file "$1" https://transfer.sh/$(basename "$1") | tee /dev/null;
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
alias transfer=transfer
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using wget
|
||||||
|
```bash
|
||||||
|
transfer() {
|
||||||
|
wget -t 1 -qO - --method=PUT --body-file="$1" --header="Content-Type: $(file -b --mime-type "$1")" https://transfer.sh/$(basename "$1");
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
alias transfer=transfer
|
||||||
|
```
|
||||||
|
|
||||||
|
## Add alias for fish-shell
|
||||||
|
|
||||||
|
### Using curl
|
||||||
|
```fish
|
||||||
|
function transfer --description 'Upload a file to transfer.sh'
|
||||||
|
if [ $argv[1] ]
|
||||||
|
# write to output to tmpfile because of progress bar
|
||||||
|
set -l tmpfile ( mktemp -t transferXXXXXX )
|
||||||
|
curl --progress-bar --upload-file "$argv[1]" https://transfer.sh/(basename $argv[1]) >> $tmpfile
|
||||||
|
cat $tmpfile
|
||||||
|
command rm -f $tmpfile
|
||||||
|
else
|
||||||
|
echo 'usage: transfer FILE_TO_TRANSFER'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
funcsave transfer
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using wget
|
||||||
|
```fish
|
||||||
|
function transfer --description 'Upload a file to transfer.sh'
|
||||||
|
if [ $argv[1] ]
|
||||||
|
wget -t 1 -qO - --method=PUT --body-file="$argv[1]" --header="Content-Type: (file -b --mime-type $argv[1])" https://transfer.sh/(basename $argv[1])
|
||||||
|
else
|
||||||
|
echo 'usage: transfer FILE_TO_TRANSFER'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
funcsave transfer
|
||||||
|
```
|
||||||
|
|
||||||
|
Now run it like this:
|
||||||
|
```bash
|
||||||
|
$ transfer test.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Add alias on Windows
|
||||||
|
|
||||||
|
Put a file called `transfer.cmd` somewhere in your PATH with this inside it:
|
||||||
|
```cmd
|
||||||
|
@echo off
|
||||||
|
setlocal
|
||||||
|
:: use env vars to pass names to PS, to avoid escaping issues
|
||||||
|
set FN=%~nx1
|
||||||
|
set FULL=%1
|
||||||
|
powershell -noprofile -command "$(Invoke-Webrequest -Method put -Infile $Env:FULL https://transfer.sh/$Env:FN).Content"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Uploading and Downloading
|
||||||
|
<a name="uploading-and-downloading"/>
|
||||||
|
|
||||||
|
### Uploading with wget
|
||||||
|
```bash
|
||||||
|
$ wget --method PUT --body-file=/tmp/file.tar https://transfer.sh/file.tar -O - -nv
|
||||||
|
```
|
||||||
|
|
||||||
|
### Uploading with PowerShell
|
||||||
|
```posh
|
||||||
|
PS H:\> invoke-webrequest -method put -infile .\file.txt https://transfer.sh/file.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Upload using HTTPie
|
||||||
|
```bash
|
||||||
|
$ http https://transfer.sh/ -vv < /tmp/test.log
|
||||||
|
```
|
||||||
|
|
||||||
|
### Uploading a filtered text file
|
||||||
|
```bash
|
||||||
|
$ grep 'pound' /var/log/syslog | curl --upload-file - https://transfer.sh/pound.log
|
||||||
|
```
|
||||||
|
|
||||||
|
### Downloading with curl
|
||||||
|
```bash
|
||||||
|
$ curl https://transfer.sh/1lDau/test.txt -o test.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Downloading with wget
|
||||||
|
```bash
|
||||||
|
$ wget https://transfer.sh/1lDau/test.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Archiving and backups
|
||||||
|
<a name="archiving-and-backups"/>
|
||||||
|
|
||||||
|
### Backup, encrypt and transfer a MySQL dump
|
||||||
|
```bash
|
||||||
|
$ mysqldump --all-databases | gzip | gpg -ac -o- | curl -X PUT --upload-file "-" https://transfer.sh/test.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Archive and upload directory
|
||||||
|
```bash
|
||||||
|
$ tar -czf - /var/log/journal | curl --upload-file - https://transfer.sh/journal.tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
### Uploading multiple files at once
|
||||||
|
```bash
|
||||||
|
$ curl -i -F filedata=@/tmp/hello.txt -F filedata=@/tmp/hello2.txt https://transfer.sh/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Combining downloads as zip or tar.gz archive
|
||||||
|
```bash
|
||||||
|
$ curl https://transfer.sh/(15HKz/hello.txt,15HKz/hello.txt).tar.gz
|
||||||
|
$ curl https://transfer.sh/(15HKz/hello.txt,15HKz/hello.txt).zip
|
||||||
|
```
|
||||||
|
|
||||||
|
### Transfer and send email with link (using an alias)
|
||||||
|
```bash
|
||||||
|
$ transfer /tmp/hello.txt | mail -s "Hello World" user@yourmaildomain.com
|
||||||
|
```
|
||||||
|
## Encrypting and decrypting
|
||||||
|
<a name="encrypting-and-decrypting"/>
|
||||||
|
|
||||||
|
### Encrypting files with password using gpg
|
||||||
|
```bash
|
||||||
|
$ cat /tmp/hello.txt | gpg -ac -o- | curl -X PUT --upload-file "-" https://transfer.sh/test.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Downloading and decrypting
|
||||||
|
```bash
|
||||||
|
$ curl https://transfer.sh/1lDau/test.txt | gpg -o- > /tmp/hello.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Import keys from [keybase](https://keybase.io/)
|
||||||
|
```bash
|
||||||
|
$ keybase track [them] # Encrypt for recipient(s)
|
||||||
|
$ cat somebackupfile.tar.gz | keybase encrypt [them] | curl --upload-file '-' https://transfer.sh/test.txt # Decrypt
|
||||||
|
$ curl https://transfer.sh/sqUFi/test.md | keybase decrypt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scanning for viruses
|
||||||
|
<a name="scanning-for-viruses"/>
|
||||||
|
|
||||||
|
### Scan for malware or viruses using Clamav
|
||||||
|
```bash
|
||||||
|
$ wget http://www.eicar.org/download/eicar.com
|
||||||
|
$ curl -X PUT --upload-file ./eicar.com https://transfer.sh/eicar.com/scan
|
||||||
|
```
|
||||||
|
|
||||||
|
### Upload malware to VirusTotal, get a permalink in return
|
||||||
|
```bash
|
||||||
|
$ curl -X PUT --upload-file nhgbhhj https://transfer.sh/test.txt/virustotal
|
||||||
|
```
|
||||||
|
## Uploading and copy download command
|
||||||
|
|
||||||
|
Download commands can be automatically copied to the clipboard after files are uploaded using transfer.sh.
|
||||||
|
|
||||||
|
It was designed for Linux or macOS.
|
||||||
|
|
||||||
|
### 1. Install xclip or xsel for Linux, macOS skips this step
|
||||||
|
|
||||||
|
- install xclip see https://command-not-found.com/xclip
|
||||||
|
|
||||||
|
- install xsel see https://command-not-found.com/xsel
|
||||||
|
|
||||||
|
Install later, add pbcopy and pbpaste to .bashrc or .zshrc or its equivalent.
|
||||||
|
|
||||||
|
- If use xclip, paste the following lines:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
alias pbcopy='xclip -selection clipboard'
|
||||||
|
alias pbpaste='xclip -selection clipboard -o'
|
||||||
|
```
|
||||||
|
|
||||||
|
- If use xsel, paste the following lines:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
alias pbcopy='xsel --clipboard --input'
|
||||||
|
alias pbpaste='xsel --clipboard --output'
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Add Uploading and copy download command shell function
|
||||||
|
|
||||||
|
1. Open .bashrc or .zshrc or its equivalent.
|
||||||
|
|
||||||
|
2. Add the following shell script:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
transfer() {
|
||||||
|
curl --progress-bar --upload-file "$1" https://transfer.sh/$(basename "$1") | pbcopy;
|
||||||
|
echo "1) Download link:"
|
||||||
|
echo "$(pbpaste)"
|
||||||
|
|
||||||
|
echo "\n2) Linux or macOS download command:"
|
||||||
|
linux_macos_download_command="wget $(pbpaste)"
|
||||||
|
echo $linux_macos_download_command
|
||||||
|
|
||||||
|
echo "\n3) Windows download command:"
|
||||||
|
windows_download_command="Invoke-WebRequest -Uri "$(pbpaste)" -OutFile $(basename $1)"
|
||||||
|
echo $windows_download_command
|
||||||
|
|
||||||
|
case $2 in
|
||||||
|
l|m) echo $linux_macos_download_command | pbcopy
|
||||||
|
;;
|
||||||
|
w) echo $windows_download_command | pbcopy
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### 3. Test
|
||||||
|
|
||||||
|
The transfer command has two parameters:
|
||||||
|
|
||||||
|
1. The first parameter is the path to upload the file.
|
||||||
|
|
||||||
|
2. The second parameter indicates which system's download command is copied. optional:
|
||||||
|
|
||||||
|
- This parameter is empty to copy the download link.
|
||||||
|
|
||||||
|
- `l` or `m` copy the Linux or macOS command that downloaded the file.
|
||||||
|
|
||||||
|
- `w` copy the Windows command that downloaded the file.
|
||||||
|
|
||||||
|
For example, The command to download the file on Windows will be copied:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ transfer ~/temp/a.log w
|
||||||
|
######################################################################## 100.0%
|
||||||
|
1) Download link:
|
||||||
|
https://transfer.sh/y0qr2c/a.log
|
||||||
|
|
||||||
|
2) Linux or macOS download command:
|
||||||
|
wget https://transfer.sh/y0qr2c/a.log
|
||||||
|
|
||||||
|
3) Windows download command:
|
||||||
|
Invoke-WebRequest -Uri https://transfer.sh/y0qr2c/a.log -OutFile a.log
|
||||||
|
```
|
||||||
41
flake.lock
generated
Normal file
41
flake.lock
generated
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1631561581,
|
||||||
|
"narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1632470817,
|
||||||
|
"narHash": "sha256-tGyOesdpqQEVqlmVeElsC98OJ2GDy+LNaCThSby/GQM=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "39e8ec2db68b863543bd377e44fbe02f8d05864e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"id": "nixpkgs",
|
||||||
|
"type": "indirect"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
212
flake.nix
Normal file
212
flake.nix
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
{
|
||||||
|
description = "Transfer.sh";
|
||||||
|
|
||||||
|
inputs.flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs, flake-utils }:
|
||||||
|
let
|
||||||
|
transfer-sh = pkgs: pkgs.buildGoModule {
|
||||||
|
src = self;
|
||||||
|
name = "transfer.sh";
|
||||||
|
vendorSha256 = "sha256-bgQUMiC33yVorcKOWhegT1/YU+fvxsz2pkeRvjf3R7g=";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
|
||||||
|
flake-utils.lib.eachDefaultSystem (
|
||||||
|
system:
|
||||||
|
let
|
||||||
|
pkgs = nixpkgs.legacyPackages.${system};
|
||||||
|
in
|
||||||
|
rec {
|
||||||
|
packages = flake-utils.lib.flattenTree {
|
||||||
|
transfer-sh = transfer-sh pkgs;
|
||||||
|
};
|
||||||
|
defaultPackage = packages.transfer-sh;
|
||||||
|
apps.transfer-sh = flake-utils.lib.mkApp { drv = packages.transfer-sh; };
|
||||||
|
defaultApp = apps.transfer-sh;
|
||||||
|
}
|
||||||
|
) // rec {
|
||||||
|
|
||||||
|
nixosModules = {
|
||||||
|
transfer-sh = { config, lib, pkgs, ... }: with lib; let
|
||||||
|
RUNTIME_DIR = "/var/lib/transfer.sh";
|
||||||
|
cfg = config.services.transfer-sh;
|
||||||
|
|
||||||
|
general_options = {
|
||||||
|
|
||||||
|
enable = mkEnableOption "Transfer.sh service";
|
||||||
|
listener = mkOption { default = 80; type = types.int; description = "port to use for http (:80)"; };
|
||||||
|
profile-listener = mkOption { default = 6060; type = types.int; description = "port to use for profiler (:6060)"; };
|
||||||
|
force-https = mkOption { type = types.nullOr types.bool; description = "redirect to https"; };
|
||||||
|
tls-listener = mkOption { default = 443; type = types.int; description = "port to use for https (:443)"; };
|
||||||
|
tls-listener-only = mkOption { type = types.nullOr types.bool; description = "flag to enable tls listener only"; };
|
||||||
|
tls-cert-file = mkOption { type = types.nullOr types.str; description = "path to tls certificate"; };
|
||||||
|
tls-private-key = mkOption { type = types.nullOr types.str; description = "path to tls private key "; };
|
||||||
|
http-auth-user = mkOption { type = types.nullOr types.str; description = "user for basic http auth on upload"; };
|
||||||
|
http-auth-pass = mkOption { type = types.nullOr types.str; description = "pass for basic http auth on upload"; };
|
||||||
|
ip-whitelist = mkOption { type = types.nullOr types.str; description = "comma separated list of ips allowed to connect to the service"; };
|
||||||
|
ip-blacklist = mkOption { type = types.nullOr types.str; description = "comma separated list of ips not allowed to connect to the service"; };
|
||||||
|
temp-path = mkOption { type = types.nullOr types.str; description = "path to temp folder"; };
|
||||||
|
web-path = mkOption { type = types.nullOr types.str; description = "path to static web files (for development or custom front end)"; };
|
||||||
|
proxy-path = mkOption { type = types.nullOr types.str; description = "path prefix when service is run behind a proxy"; };
|
||||||
|
proxy-port = mkOption { type = types.nullOr types.str; description = "port of the proxy when the service is run behind a proxy"; };
|
||||||
|
ga-key = mkOption { type = types.nullOr types.str; description = "google analytics key for the front end"; };
|
||||||
|
email-contact = mkOption { type = types.nullOr types.str; description = "email contact for the front end"; };
|
||||||
|
uservoice-key = mkOption { type = types.nullOr types.str; description = "user voice key for the front end"; };
|
||||||
|
lets-encrypt-hosts = mkOption { type = types.nullOr (types.listOf types.str); description = "hosts to use for lets encrypt certificates"; };
|
||||||
|
log = mkOption { type = types.nullOr types.str; description = "path to log file"; };
|
||||||
|
cors-domains = mkOption { type = types.nullOr (types.listOf types.str); description = "comma separated list of domains for CORS, setting it enable CORS "; };
|
||||||
|
clamav-host = mkOption { type = types.nullOr types.str; description = "host for clamav feature"; };
|
||||||
|
rate-limit = mkOption { type = types.nullOr types.int; description = "request per minute"; };
|
||||||
|
max-upload-size = mkOption { type = types.nullOr types.int; description = "max upload size in kilobytes "; };
|
||||||
|
purge-days = mkOption { type = types.nullOr types.int; description = "number of days after the uploads are purged automatically "; };
|
||||||
|
random-token-length = mkOption { type = types.nullOr types.int; description = "length of the random token for the upload path (double the size for delete path)"; };
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
provider_options = {
|
||||||
|
|
||||||
|
aws = {
|
||||||
|
enable = mkEnableOption "Enable AWS backend";
|
||||||
|
aws-access-key = mkOption { type = types.str; description = "aws access key"; };
|
||||||
|
aws-secret-key = mkOption { type = types.str; description = "aws secret key"; };
|
||||||
|
bucket = mkOption { type = types.str; description = "aws bucket "; };
|
||||||
|
s3-endpoint = mkOption {
|
||||||
|
type = types.nullOr types.str;
|
||||||
|
description = ''
|
||||||
|
Custom S3 endpoint.
|
||||||
|
If you specify the s3-region, you don't need to set the endpoint URL since the correct endpoint will used automatically.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
s3-region = mkOption { type = types.str; description = "region of the s3 bucket eu-west-"; };
|
||||||
|
s3-no-multipart = mkOption { type = types.nullOr types.bool; description = "disables s3 multipart upload "; };
|
||||||
|
s3-path-style = mkOption { type = types.nullOr types.str; description = "Forces path style URLs, required for Minio. "; };
|
||||||
|
};
|
||||||
|
|
||||||
|
storj = {
|
||||||
|
enable = mkEnableOption "Enable storj backend";
|
||||||
|
storj-access = mkOption { type = types.str; description = "Access for the project"; };
|
||||||
|
storj-bucket = mkOption { type = types.str; description = "Bucket to use within the project"; };
|
||||||
|
};
|
||||||
|
|
||||||
|
gdrive = {
|
||||||
|
enable = mkEnableOption "Enable gdrive backend";
|
||||||
|
gdrive-client-json = mkOption { type = types.str; description = "oauth client json config for gdrive provider"; };
|
||||||
|
gdrive-chunk-size = mkOption { default = 8; type = types.nullOr types.int; description = "chunk size for gdrive upload in megabytes, must be lower than available memory (8 MB)"; };
|
||||||
|
basedir = mkOption { type = types.str; description = "path storage for gdrive provider"; default = "${cfg.stateDir}/store"; };
|
||||||
|
purge-interval = mkOption { type = types.nullOr types.int; description = "interval in hours to run the automatic purge for (not applicable to S3 and Storj)"; };
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
local = {
|
||||||
|
enable = mkEnableOption "Enable local backend";
|
||||||
|
basedir = mkOption { type = types.str; description = "path storage for local provider"; default = "${cfg.stateDir}/store"; };
|
||||||
|
purge-interval = mkOption { type = types.nullOr types.int; description = "interval in hours to run the automatic purge for (not applicable to S3 and Storj)"; };
|
||||||
|
};
|
||||||
|
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.services.transfer-sh = fold recursiveUpdate {} [
|
||||||
|
general_options
|
||||||
|
{
|
||||||
|
provider = provider_options;
|
||||||
|
user = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
description = "User to run the service under";
|
||||||
|
default = "transfer.sh";
|
||||||
|
};
|
||||||
|
group = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
description = "Group to run the service under";
|
||||||
|
default = "transfer.sh";
|
||||||
|
};
|
||||||
|
stateDir = mkOption {
|
||||||
|
type = types.path;
|
||||||
|
description = "Variable state directory";
|
||||||
|
default = RUNTIME_DIR;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
config = let
|
||||||
|
|
||||||
|
mkFlags = cfg: options:
|
||||||
|
let
|
||||||
|
mkBoolFlag = option: if cfg.${option} then [ "--${option}" ] else [];
|
||||||
|
mkFlag = option:
|
||||||
|
if isBool cfg.${option}
|
||||||
|
then mkBoolFlag option
|
||||||
|
else [ "--${option}" "${cfg.${option}}" ];
|
||||||
|
|
||||||
|
in
|
||||||
|
lists.flatten (map (mkFlag) (filter (option: cfg.${option} != null && option != "enable") options));
|
||||||
|
|
||||||
|
aws-config = (mkFlags cfg.provider.aws (attrNames provider_options)) ++ [ "--provider" "aws" ];
|
||||||
|
gdrive-config = mkFlags cfg.provider.gdrive (attrNames provider_options.gdrive) ++ [ "--provider" "gdrive" ];
|
||||||
|
storj-config = mkFlags cfg.provider.storj (attrNames provider_options.storj) ++ [ "--provider" "storj" ];
|
||||||
|
local-config = mkFlags cfg.provider.local (attrNames provider_options.local) ++ [ "--provider" "local" ];
|
||||||
|
|
||||||
|
general-config = concatStringsSep " " (mkFlags cfg (attrNames general_options));
|
||||||
|
provider-config = concatStringsSep " " (
|
||||||
|
if cfg.provider.aws.enable && !cfg.provider.storj.enable && !cfg.provider.gdrive.enable && !cfg.provider.local.enable then aws-config
|
||||||
|
else if !cfg.provider.aws.enable && cfg.provider.storj.enable && !cfg.provider.gdrive.enable && !cfg.provider.local.enable then storj-config
|
||||||
|
else if !cfg.provider.aws.enable && !cfg.provider.storj.enable && cfg.provider.gdrive.enable && !cfg.provider.local.enable then gdrive-config
|
||||||
|
else if !cfg.provider.aws.enable && !cfg.provider.storj.enable && !cfg.provider.gdrive.enable && cfg.provider.local.enable then local-config
|
||||||
|
else throw "transfer.sh requires exactly one provider (aws, storj, gdrive, local)"
|
||||||
|
);
|
||||||
|
|
||||||
|
in
|
||||||
|
lib.mkIf cfg.enable
|
||||||
|
{
|
||||||
|
systemd.tmpfiles.rules = [
|
||||||
|
"d ${cfg.stateDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||||
|
] ++ optional cfg.provider.gdrive.enable cfg.provider.gdrive.basedir
|
||||||
|
++ optional cfg.provider.local.enable cfg.provider.local.basedir;
|
||||||
|
|
||||||
|
systemd.services.transfer-sh = {
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
after = [ "network.target" ];
|
||||||
|
serviceConfig = {
|
||||||
|
User = cfg.user;
|
||||||
|
Group = cfg.group;
|
||||||
|
ExecStart = "${transfer-sh pkgs}/bin/transfer.sh ${general-config} ${provider-config} ";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
networking.firewall.allowedTCPPorts = [ cfg.listener cfg.profile-listener cfg.tls-listener ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
default = { self, pkgs, ... }: {
|
||||||
|
imports = [ nixosModules.transfer-sh ];
|
||||||
|
# Network configuration.
|
||||||
|
|
||||||
|
# useDHCP is generally considered to better be turned off in favor
|
||||||
|
# of <adapter>.useDHCP
|
||||||
|
networking.useDHCP = false;
|
||||||
|
networking.firewall.allowedTCPPorts = [];
|
||||||
|
|
||||||
|
# Enable the inventaire server.
|
||||||
|
services.transfer-sh = {
|
||||||
|
enable = true;
|
||||||
|
provider.local = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
nixpkgs.config.allowUnfree = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
nixosConfigurations."container" = nixpkgs.lib.nixosSystem {
|
||||||
|
system = "x86_64-linux";
|
||||||
|
modules = [
|
||||||
|
nixosModules.default
|
||||||
|
({ ... }: { boot.isContainer = true; })
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
};
|
||||||
|
}
|
||||||
36
go.mod
Normal file
36
go.mod
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
module github.com/dutchcoders/transfer.sh
|
||||||
|
|
||||||
|
go 1.13
|
||||||
|
|
||||||
|
require (
|
||||||
|
cloud.google.com/go v0.77.0 // indirect
|
||||||
|
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14
|
||||||
|
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2
|
||||||
|
github.com/aws/aws-sdk-go v1.37.14
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
||||||
|
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
|
||||||
|
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329
|
||||||
|
github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3
|
||||||
|
github.com/elazarl/go-bindata-assetfs v1.0.1
|
||||||
|
github.com/fatih/color v1.10.0
|
||||||
|
github.com/garyburd/redigo v1.6.2 // indirect
|
||||||
|
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
|
||||||
|
github.com/gorilla/handlers v1.5.1
|
||||||
|
github.com/gorilla/mux v1.8.0
|
||||||
|
github.com/gorilla/securecookie v1.1.1 // indirect
|
||||||
|
github.com/microcosm-cc/bluemonday v1.0.16
|
||||||
|
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0
|
||||||
|
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||||
|
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce
|
||||||
|
github.com/urfave/cli v1.22.5
|
||||||
|
go.opencensus.io v0.22.6 // indirect
|
||||||
|
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
|
||||||
|
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
|
||||||
|
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
|
||||||
|
google.golang.org/api v0.40.0
|
||||||
|
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 // indirect
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
|
||||||
|
storj.io/common v0.0.0-20220131120956-e74f624a3d55
|
||||||
|
storj.io/uplink v1.7.2-0.20220131124001-c1db742c840d
|
||||||
|
)
|
||||||
746
go.sum
Normal file
746
go.sum
Normal file
@ -0,0 +1,746 @@
|
|||||||
|
cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||||
|
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||||
|
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||||
|
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||||
|
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||||
|
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||||
|
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||||
|
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||||
|
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||||
|
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||||
|
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||||
|
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||||
|
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||||
|
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||||
|
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
|
||||||
|
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||||
|
cloud.google.com/go v0.77.0 h1:qA5V5+uQf6Mgr+tmFI8UT3D/ELyhIYkPwNGao/3Y+sQ=
|
||||||
|
cloud.google.com/go v0.77.0/go.mod h1:R8fYSLIilC247Iu8WS2OGHw1E/Ufn7Pd7HiDjTqiURs=
|
||||||
|
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||||
|
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||||
|
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||||
|
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||||
|
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||||
|
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||||
|
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||||
|
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||||
|
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||||
|
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||||
|
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||||
|
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||||
|
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||||
|
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||||
|
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||||
|
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||||
|
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||||
|
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||||
|
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||||
|
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||||
|
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||||
|
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||||
|
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||||
|
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14 h1:3zOOc7WdrATDXof+h/rBgMsg0sAmZIEVHft1UbWHh94=
|
||||||
|
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14/go.mod h1:+VFiaivV54Sa94ijzA/ZHQLoHuoUIS9hIqCK6f/76Zw=
|
||||||
|
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2 h1:sIvihcW4qpN5qGSjmrsDDAbLpEq5tuHjJJfWY0Hud5Y=
|
||||||
|
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2/go.mod h1:3YwJE8rEisS9eraee0hygGG4G3gqX8H8Nyu+nPTUnGU=
|
||||||
|
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||||
|
github.com/aws/aws-sdk-go v1.37.14 h1:thuR1hd1doCvsaMDYDMhqCGSmw39bSvZaw+DPGhMm5w=
|
||||||
|
github.com/aws/aws-sdk-go v1.37.14/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||||
|
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||||
|
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||||
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||||
|
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||||
|
github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
|
||||||
|
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||||
|
github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo=
|
||||||
|
github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI=
|
||||||
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||||
|
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||||
|
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||||
|
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||||
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
|
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||||
|
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||||
|
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||||
|
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||||
|
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e h1:rcHHSQqzCgvlwP0I/fQ8rQMn/MpHE5gWSLdtpxtP6KQ=
|
||||||
|
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e/go.mod h1:Byz7q8MSzSPkouskHJhX0er2mZY/m0Vj5bMeMCkkyY4=
|
||||||
|
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329 h1:ERqCkG/uSyT74P1m/j9yR+so+7ynY4fbTvLY/Mr1ZMg=
|
||||||
|
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329/go.mod h1:G5qOfE5bQZ5scycLpB7fYWgN4y3xdfXo+pYWM8z2epY=
|
||||||
|
github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3 h1:HyfU90/8y9S5IkHTQgIfzs4dT3iagbJUJLncCsSwZCI=
|
||||||
|
github.com/dutchcoders/transfer.sh-web v0.0.0-20211215083008-31e11925a9d3/go.mod h1:F6Q37CxDh2MHr5KXkcZmNB3tdkK7v+bgE+OpBY+9ilI=
|
||||||
|
github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
|
||||||
|
github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
|
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
|
||||||
|
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
|
||||||
|
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
|
||||||
|
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||||
|
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||||
|
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||||
|
github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
|
||||||
|
github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM=
|
||||||
|
github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
|
||||||
|
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||||
|
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||||
|
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||||
|
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||||
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||||
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||||
|
github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||||
|
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||||
|
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||||
|
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||||
|
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||||
|
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg=
|
||||||
|
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4=
|
||||||
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
|
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
|
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
|
github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||||
|
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||||
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
|
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
|
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||||
|
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||||
|
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||||
|
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||||
|
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||||
|
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||||
|
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||||
|
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||||
|
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||||
|
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||||
|
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
|
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||||
|
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||||
|
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
|
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
|
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
|
github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||||
|
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||||
|
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||||
|
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||||
|
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||||
|
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||||
|
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||||
|
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||||
|
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
|
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
|
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
|
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
|
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
|
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||||
|
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||||
|
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||||
|
github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk=
|
||||||
|
github.com/google/pprof v0.0.0-20211108044417-e9b028704de0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||||
|
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||||
|
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
|
||||||
|
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||||
|
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||||
|
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
|
||||||
|
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
|
||||||
|
github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
|
||||||
|
github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
|
||||||
|
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||||
|
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||||
|
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
|
||||||
|
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||||
|
github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||||
|
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||||
|
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||||
|
github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
|
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||||
|
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||||
|
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||||
|
github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o=
|
||||||
|
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
|
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||||
|
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||||
|
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||||
|
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
|
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||||
|
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0=
|
||||||
|
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||||
|
github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||||
|
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||||
|
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||||
|
github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||||
|
github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
|
||||||
|
github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
|
||||||
|
github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||||
|
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
|
||||||
|
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||||
|
github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||||
|
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||||
|
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||||
|
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||||
|
github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
|
||||||
|
github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||||
|
github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||||
|
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||||
|
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
|
||||||
|
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
|
||||||
|
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||||
|
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||||
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||||
|
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||||
|
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||||
|
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
|
||||||
|
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||||
|
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||||
|
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||||
|
github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
|
||||||
|
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||||
|
github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||||
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||||
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||||
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
|
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||||
|
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||||
|
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||||
|
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
|
||||||
|
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||||
|
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||||
|
github.com/shuLhan/go-bindata v4.0.0+incompatible/go.mod h1:pkcPAATLBDD2+SpAPnX5vEM90F7fcwHCvvLCMXcmw3g=
|
||||||
|
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||||
|
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||||
|
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||||
|
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||||
|
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||||
|
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||||
|
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||||
|
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||||
|
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||||
|
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||||
|
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||||
|
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||||
|
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||||
|
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||||
|
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||||
|
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||||
|
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||||
|
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||||
|
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||||
|
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||||
|
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||||
|
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
|
||||||
|
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
|
||||||
|
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||||
|
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||||
|
github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes=
|
||||||
|
github.com/spacemonkeygo/monkit/v3 v3.0.17 h1:rqIuLhRUr2UtS3WNVbPY/BwvjlwKVvSOVY5p0QVocxE=
|
||||||
|
github.com/spacemonkeygo/monkit/v3 v3.0.17/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
|
||||||
|
github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo=
|
||||||
|
github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||||
|
github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||||
|
github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||||
|
github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||||
|
github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||||
|
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
|
||||||
|
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
|
||||||
|
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
|
||||||
|
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
|
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||||
|
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||||
|
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
|
||||||
|
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
|
||||||
|
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||||
|
github.com/zeebo/admission/v3 v3.0.3/go.mod h1:2OWyAS5yo0Xvj2AEUosOjTUHxaY0oIIiCrXGKCYzWpo=
|
||||||
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
|
github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
|
||||||
|
github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||||
|
github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo=
|
||||||
|
github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w=
|
||||||
|
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||||
|
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||||
|
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||||
|
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||||
|
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||||
|
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||||
|
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||||
|
go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok=
|
||||||
|
go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||||
|
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||||
|
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||||
|
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
|
||||||
|
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
|
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
|
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||||
|
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||||
|
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||||
|
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||||
|
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||||
|
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||||
|
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||||
|
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||||
|
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||||
|
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
|
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
|
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||||
|
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||||
|
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||||
|
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||||
|
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||||
|
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||||
|
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||||
|
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||||
|
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||||
|
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||||
|
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||||
|
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||||
|
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||||
|
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||||
|
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||||
|
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||||
|
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||||
|
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||||
|
golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||||
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20211109065445-02f5c0300f6e h1:i6Vklmyu+fZMFYpum+sR4ZWABGW7MyIxfJZXYvcnbns=
|
||||||
|
golang.org/x/sys v0.0.0-20211109065445-02f5c0300f6e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||||
|
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
|
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
|
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
|
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
|
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||||
|
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||||
|
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||||
|
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||||
|
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||||
|
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||||
|
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||||
|
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||||
|
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||||
|
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||||
|
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||||
|
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||||
|
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||||
|
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||||
|
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||||
|
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||||
|
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||||
|
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||||
|
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||||
|
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||||
|
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||||
|
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||||
|
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||||
|
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||||
|
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||||
|
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||||
|
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||||
|
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||||
|
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||||
|
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
|
||||||
|
google.golang.org/api v0.40.0 h1:uWrpz12dpVPn7cojP82mk02XDgTJLDPc2KbVTxrWb4A=
|
||||||
|
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
|
||||||
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||||
|
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||||
|
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||||
|
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||||
|
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||||
|
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||||
|
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||||
|
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||||
|
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||||
|
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||||
|
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||||
|
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||||
|
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
|
||||||
|
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||||
|
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||||
|
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||||
|
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||||
|
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20210212180131-e7f2df4ecc2d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 h1:Px6YyLaNKEo5eoniIBAv6Es0jbvyEmSYqOac64iS2Rs=
|
||||||
|
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||||
|
google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||||
|
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||||
|
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||||
|
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||||
|
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||||
|
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||||
|
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||||
|
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||||
|
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||||
|
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||||
|
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||||
|
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||||
|
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||||
|
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||||
|
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||||
|
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||||
|
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
|
||||||
|
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||||
|
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||||
|
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||||
|
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||||
|
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||||
|
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||||
|
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||||
|
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||||
|
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||||
|
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||||
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||||
|
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||||
|
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||||
|
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||||
|
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||||
|
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||||
|
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||||
|
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||||
|
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||||
|
storj.io/common v0.0.0-20220131120956-e74f624a3d55 h1:HgFcPy1vohRJ4VRRGej92N31i5znL1KGWhDVgGgjBG4=
|
||||||
|
storj.io/common v0.0.0-20220131120956-e74f624a3d55/go.mod h1:m0489td5+rKDdoiYOzCkh3CfGW/cLyntZiYfso+QfMs=
|
||||||
|
storj.io/drpc v0.0.29 h1:Ihd4ls/JQFr0lctefie3iu+3QM4duccCKr9uMzf4sKY=
|
||||||
|
storj.io/drpc v0.0.29/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
|
||||||
|
storj.io/uplink v1.7.2-0.20220131124001-c1db742c840d h1:Fb9Tht/h8KJbn02obTM1mseR4b7WQ7PCf+Nc3wXC3vk=
|
||||||
|
storj.io/uplink v1.7.2-0.20220131124001-c1db742c840d/go.mod h1:BYnzl+ordVACxbWM7eKUvkqC+GvSnMmVbIyA2BNGBmk=
|
||||||
203
lock.json
203
lock.json
@ -1,203 +0,0 @@
|
|||||||
{
|
|
||||||
"memo": "5b27aecb0272e40f3b8b8f9a5deeb7c9f5dbf06c53b1134de2b84eac466d27e0",
|
|
||||||
"projects": [
|
|
||||||
{
|
|
||||||
"name": "github.com/PuerkitoBio/ghost",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "206e6e460e14a42d1d811c970b30248db058e9b2",
|
|
||||||
"packages": [
|
|
||||||
".",
|
|
||||||
"handlers"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/VojtechVitek/ratelimit",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "dc172bc0f6d241e980010dbc63957ef1a2c8ca33",
|
|
||||||
"packages": [
|
|
||||||
".",
|
|
||||||
"memory"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/dutchcoders/go-clamd",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "a9a81beaffff0392094052913ec45fa140eb8511",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/dutchcoders/go-virustotal",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "24cc8e6fa329f020c70a3b32330b5743f1ba7971",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/dutchcoders/transfer.sh-web",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "cdb97c4b64a33dd1eec7e94b2c200cd9b06296a1",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/elazarl/go-bindata-assetfs",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/fatih/color",
|
|
||||||
"version": "v1.4.1",
|
|
||||||
"revision": "9131ab34cf20d2f6d83fdc67168a5430d1c7dc23",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/garyburd/redigo",
|
|
||||||
"version": "v1.0.0",
|
|
||||||
"revision": "8873b2f1995f59d4bcdd2b0dc9858e2cb9bf0c13",
|
|
||||||
"packages": [
|
|
||||||
"internal",
|
|
||||||
"redis"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/goamz/goamz",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "c35091c30f44b7f151ec9028b895465a191d1ea7",
|
|
||||||
"packages": [
|
|
||||||
"aws",
|
|
||||||
"s3"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/golang/gddo",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "72302b972abba39585150723aea3cf343e99437c",
|
|
||||||
"packages": [
|
|
||||||
"httputil/header"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/gorilla/context",
|
|
||||||
"version": "v1.1",
|
|
||||||
"revision": "1ea25387ff6f684839d82767c1733ff4d4d15d0a",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/gorilla/mux",
|
|
||||||
"version": "v1.3.0",
|
|
||||||
"revision": "392c28fe23e1c45ddba891b0320b3b5df220beea",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/gorilla/securecookie",
|
|
||||||
"version": "v1.1",
|
|
||||||
"revision": "667fe4e3466a040b780561fe9b51a83a3753eefc",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/mattn/go-colorable",
|
|
||||||
"version": "v0.0.7",
|
|
||||||
"revision": "d228849504861217f796da67fae4f6e347643f15",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/mattn/go-isatty",
|
|
||||||
"version": "v0.0.1",
|
|
||||||
"revision": "3a115632dcd687f9c8cd01679c83a06a0e21c1f3",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/minio/cli",
|
|
||||||
"version": "v1.3.0",
|
|
||||||
"revision": "8683fa7fef37cc8cb092f47bdb6b403e0049f9ee",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/nu7hatch/gouuid",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "179d4d0c4d8d407a32af483c2354df1d2c91e6c3",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/russross/blackfriday",
|
|
||||||
"version": "v1.4",
|
|
||||||
"revision": "0b647d0506a698cca42caca173e55559b12a69f2",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/shurcooL/sanitized_anchor_name",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "1dba4b3954bc059efc3991ec364f9f9a35f597d2",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "github.com/vaughan0/go-ini",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "golang.org/x/crypto",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7",
|
|
||||||
"packages": [
|
|
||||||
"acme",
|
|
||||||
"acme/autocert"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "golang.org/x/net",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "a6577fac2d73be281a500b310739095313165611",
|
|
||||||
"packages": [
|
|
||||||
"context",
|
|
||||||
"context/ctxhttp"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "golang.org/x/sys",
|
|
||||||
"branch": "master",
|
|
||||||
"revision": "99f16d856c9836c42d24e7ab64ea72916925fa97",
|
|
||||||
"packages": [
|
|
||||||
"unix"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "gopkg.in/check.v1",
|
|
||||||
"branch": "v1",
|
|
||||||
"revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
|
|
||||||
"packages": [
|
|
||||||
"."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
12
main.go
12
main.go
@ -1,8 +1,16 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import "github.com/dutchcoders/transfer.sh/cmd"
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/dutchcoders/transfer.sh/cmd"
|
||||||
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
app := cmd.New()
|
app := cmd.New()
|
||||||
app.RunAndExitOnError()
|
err := app.Run(os.Args)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,8 @@
|
|||||||
The MIT License (MIT)
|
The MIT License (MIT)
|
||||||
|
|
||||||
Copyright (c) 2014-2017 DutchCoders [https://github.com/dutchcoders/]
|
Copyright (c) 2014-2017 DutchCoders [https://github.com/dutchcoders/]
|
||||||
|
Copyright (c) 2018-2020 Andrea Spacca.
|
||||||
|
Copyright (c) 2020- Andrea Spacca and Stefan Benten.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
@ -25,20 +27,19 @@ THE SOFTWARE.
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
// _ "transfer.sh/app/handlers"
|
"errors"
|
||||||
// _ "transfer.sh/app/utils"
|
|
||||||
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"io/ioutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
clamd "github.com/dutchcoders/go-clamd"
|
clamd "github.com/dutchcoders/go-clamd"
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const clamavScanStatusOK = "OK"
|
||||||
|
|
||||||
func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
|
|
||||||
@ -47,28 +48,55 @@ func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
contentLength := r.ContentLength
|
contentLength := r.ContentLength
|
||||||
contentType := r.Header.Get("Content-Type")
|
contentType := r.Header.Get("Content-Type")
|
||||||
|
|
||||||
log.Printf("Scanning %s %d %s", filename, contentLength, contentType)
|
s.logger.Printf("Scanning %s %d %s", filename, contentLength, contentType)
|
||||||
|
|
||||||
var reader io.Reader
|
file, err := ioutil.TempFile(s.tempPath, "clamav-")
|
||||||
|
defer s.cleanTmpFile(file)
|
||||||
reader = r.Body
|
|
||||||
|
|
||||||
c := clamd.NewClamd(s.ClamAVDaemonHost)
|
|
||||||
|
|
||||||
abort := make(chan bool)
|
|
||||||
response, err := c.ScanStream(reader, abort)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("%s", err.Error())
|
s.logger.Printf("%s", err.Error())
|
||||||
http.Error(w, err.Error(), 500)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
_, err = io.Copy(file, r.Body)
|
||||||
case s := <-response:
|
if err != nil {
|
||||||
w.Write([]byte(fmt.Sprintf("%v\n", s.Status)))
|
s.logger.Printf("%s", err.Error())
|
||||||
case <-time.After(time.Second * 60):
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
abort <- true
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
close(abort)
|
status, err := s.performScan(file.Name())
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Printf("%s", err.Error())
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", status)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) performScan(path string) (string, error) {
|
||||||
|
c := clamd.NewClamd(s.ClamAVDaemonHost)
|
||||||
|
|
||||||
|
responseCh := make(chan chan *clamd.ScanResult)
|
||||||
|
errCh := make(chan error)
|
||||||
|
go func(responseCh chan chan *clamd.ScanResult, errCh chan error) {
|
||||||
|
response, err := c.ScanFile(path)
|
||||||
|
if err != nil {
|
||||||
|
errCh <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
responseCh <- response
|
||||||
|
}(responseCh, errCh)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-errCh:
|
||||||
|
return "", err
|
||||||
|
case response := <-responseCh:
|
||||||
|
st := <-response
|
||||||
|
return st.Status, nil
|
||||||
|
case <-time.After(time.Second * 60):
|
||||||
|
return "", errors.New("clamav scan timeout")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,65 +0,0 @@
|
|||||||
/*
|
|
||||||
https://github.com/fs111/kurz.go/blob/master/src/codec.go
|
|
||||||
|
|
||||||
Originally written and Copyright (c) 2011 André Kelpe
|
|
||||||
Modifications Copyright (c) 2015 John Ko
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
|
||||||
the Software without restriction, including without limitation the rights to
|
|
||||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
||||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
|
||||||
subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
||||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
||||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
||||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
||||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// characters used for short-urls
|
|
||||||
SYMBOLS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
|
||||||
|
|
||||||
// someone set us up the bomb !!
|
|
||||||
BASE = int64(len(SYMBOLS))
|
|
||||||
)
|
|
||||||
|
|
||||||
// encodes a number into our *base* representation
|
|
||||||
// TODO can this be made better with some bitshifting?
|
|
||||||
func Encode(number int64) string {
|
|
||||||
rest := number % BASE
|
|
||||||
// strings are a bit weird in go...
|
|
||||||
result := string(SYMBOLS[rest])
|
|
||||||
if number-rest != 0 {
|
|
||||||
newnumber := (number - rest) / BASE
|
|
||||||
result = Encode(newnumber) + result
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decodes a string given in our encoding and returns the decimal
|
|
||||||
// integer.
|
|
||||||
func Decode(input string) int64 {
|
|
||||||
const floatbase = float64(BASE)
|
|
||||||
l := len(input)
|
|
||||||
var sum int = 0
|
|
||||||
for index := l - 1; index > -1; index -= 1 {
|
|
||||||
current := string(input[index])
|
|
||||||
pos := strings.Index(SYMBOLS, current)
|
|
||||||
sum = sum + (pos * int(math.Pow(floatbase, float64((l-index-1)))))
|
|
||||||
}
|
|
||||||
return int64(sum)
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -13,16 +13,16 @@ import (
|
|||||||
func Test(t *testing.T) { TestingT(t) }
|
func Test(t *testing.T) { TestingT(t) }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
_ = Suite(&SuiteRedirectWithForceHTTPs{})
|
_ = Suite(&suiteRedirectWithForceHTTPS{})
|
||||||
_ = Suite(&SuiteRedirectWithoutForceHTTPs{})
|
_ = Suite(&suiteRedirectWithoutForceHTTPS{})
|
||||||
)
|
)
|
||||||
|
|
||||||
type SuiteRedirectWithForceHTTPs struct {
|
type suiteRedirectWithForceHTTPS struct {
|
||||||
handler http.HandlerFunc
|
handler http.HandlerFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithForceHTTPs) SetUpTest(c *C) {
|
func (s *suiteRedirectWithForceHTTPS) SetUpTest(c *C) {
|
||||||
srvr, err := New(ForceHTTPs())
|
srvr, err := New(ForceHTTPS())
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@ -32,7 +32,7 @@ func (s *SuiteRedirectWithForceHTTPs) SetUpTest(c *C) {
|
|||||||
s.handler = srvr.RedirectHandler(handler)
|
s.handler = srvr.RedirectHandler(handler)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithForceHTTPs) TestHTTPs(c *C) {
|
func (s *suiteRedirectWithForceHTTPS) TestHTTPs(c *C) {
|
||||||
req := httptest.NewRequest("GET", "https://test/test", nil)
|
req := httptest.NewRequest("GET", "https://test/test", nil)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@ -42,7 +42,7 @@ func (s *SuiteRedirectWithForceHTTPs) TestHTTPs(c *C) {
|
|||||||
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithForceHTTPs) TestOnion(c *C) {
|
func (s *suiteRedirectWithForceHTTPS) TestOnion(c *C) {
|
||||||
req := httptest.NewRequest("GET", "http://test.onion/test", nil)
|
req := httptest.NewRequest("GET", "http://test.onion/test", nil)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@ -52,7 +52,7 @@ func (s *SuiteRedirectWithForceHTTPs) TestOnion(c *C) {
|
|||||||
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithForceHTTPs) TestXForwardedFor(c *C) {
|
func (s *suiteRedirectWithForceHTTPS) TestXForwardedFor(c *C) {
|
||||||
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
req.Header.Set("X-Forwarded-Proto", "https")
|
||||||
|
|
||||||
@ -63,7 +63,7 @@ func (s *SuiteRedirectWithForceHTTPs) TestXForwardedFor(c *C) {
|
|||||||
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithForceHTTPs) TestHTTP(c *C) {
|
func (s *suiteRedirectWithForceHTTPS) TestHTTP(c *C) {
|
||||||
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@ -74,11 +74,11 @@ func (s *SuiteRedirectWithForceHTTPs) TestHTTP(c *C) {
|
|||||||
c.Assert(resp.Header.Get("Location"), Equals, "https://127.0.0.1/test")
|
c.Assert(resp.Header.Get("Location"), Equals, "https://127.0.0.1/test")
|
||||||
}
|
}
|
||||||
|
|
||||||
type SuiteRedirectWithoutForceHTTPs struct {
|
type suiteRedirectWithoutForceHTTPS struct {
|
||||||
handler http.HandlerFunc
|
handler http.HandlerFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithoutForceHTTPs) SetUpTest(c *C) {
|
func (s *suiteRedirectWithoutForceHTTPS) SetUpTest(c *C) {
|
||||||
srvr, err := New()
|
srvr, err := New()
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
@ -89,7 +89,7 @@ func (s *SuiteRedirectWithoutForceHTTPs) SetUpTest(c *C) {
|
|||||||
s.handler = srvr.RedirectHandler(handler)
|
s.handler = srvr.RedirectHandler(handler)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithoutForceHTTPs) TestHTTP(c *C) {
|
func (s *suiteRedirectWithoutForceHTTPS) TestHTTP(c *C) {
|
||||||
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
req := httptest.NewRequest("GET", "http://127.0.0.1/test", nil)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@ -99,7 +99,7 @@ func (s *SuiteRedirectWithoutForceHTTPs) TestHTTP(c *C) {
|
|||||||
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
c.Assert(resp.StatusCode, Equals, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SuiteRedirectWithoutForceHTTPs) TestHTTPs(c *C) {
|
func (s *suiteRedirectWithoutForceHTTPS) TestHTTPs(c *C) {
|
||||||
req := httptest.NewRequest("GET", "https://127.0.0.1/test", nil)
|
req := httptest.NewRequest("GET", "https://127.0.0.1/test", nil)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|||||||
212
server/ip_filter.go
Normal file
212
server/ip_filter.go
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
/*
|
||||||
|
MIT License
|
||||||
|
Copyright © 2016 <dev@jpillora.com>
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/tomasen/realip"
|
||||||
|
)
|
||||||
|
|
||||||
|
//IPFilterOptions for ipFilter. Allowed takes precedence over Blocked.
|
||||||
|
//IPs can be IPv4 or IPv6 and can optionally contain subnet
|
||||||
|
//masks (/24). Note however, determining if a given IP is
|
||||||
|
//included in a subnet requires a linear scan so is less performant
|
||||||
|
//than looking up single IPs.
|
||||||
|
//
|
||||||
|
//This could be improved with some algorithmic magic.
|
||||||
|
type IPFilterOptions struct {
|
||||||
|
//explicity allowed IPs
|
||||||
|
AllowedIPs []string
|
||||||
|
//explicity blocked IPs
|
||||||
|
BlockedIPs []string
|
||||||
|
//block by default (defaults to allow)
|
||||||
|
BlockByDefault bool
|
||||||
|
// TrustProxy enable check request IP from proxy
|
||||||
|
TrustProxy bool
|
||||||
|
|
||||||
|
Logger interface {
|
||||||
|
Printf(format string, v ...interface{})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ipFilter
|
||||||
|
type ipFilter struct {
|
||||||
|
opts IPFilterOptions
|
||||||
|
//mut protects the below
|
||||||
|
//rw since writes are rare
|
||||||
|
mut sync.RWMutex
|
||||||
|
defaultAllowed bool
|
||||||
|
ips map[string]bool
|
||||||
|
subnets []*subnet
|
||||||
|
}
|
||||||
|
|
||||||
|
type subnet struct {
|
||||||
|
str string
|
||||||
|
ipnet *net.IPNet
|
||||||
|
allowed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIPFilter(opts IPFilterOptions) *ipFilter {
|
||||||
|
if opts.Logger == nil {
|
||||||
|
flags := log.LstdFlags
|
||||||
|
opts.Logger = log.New(os.Stdout, "", flags)
|
||||||
|
}
|
||||||
|
f := &ipFilter{
|
||||||
|
opts: opts,
|
||||||
|
ips: map[string]bool{},
|
||||||
|
defaultAllowed: !opts.BlockByDefault,
|
||||||
|
}
|
||||||
|
for _, ip := range opts.BlockedIPs {
|
||||||
|
f.BlockIP(ip)
|
||||||
|
}
|
||||||
|
for _, ip := range opts.AllowedIPs {
|
||||||
|
f.AllowIP(ip)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *ipFilter) AllowIP(ip string) bool {
|
||||||
|
return f.ToggleIP(ip, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *ipFilter) BlockIP(ip string) bool {
|
||||||
|
return f.ToggleIP(ip, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *ipFilter) ToggleIP(str string, allowed bool) bool {
|
||||||
|
//check if has subnet
|
||||||
|
if ip, net, err := net.ParseCIDR(str); err == nil {
|
||||||
|
// containing only one ip?
|
||||||
|
if n, total := net.Mask.Size(); n == total {
|
||||||
|
f.mut.Lock()
|
||||||
|
f.ips[ip.String()] = allowed
|
||||||
|
f.mut.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
//check for existing
|
||||||
|
f.mut.Lock()
|
||||||
|
found := false
|
||||||
|
for _, subnet := range f.subnets {
|
||||||
|
if subnet.str == str {
|
||||||
|
found = true
|
||||||
|
subnet.allowed = allowed
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
f.subnets = append(f.subnets, &subnet{
|
||||||
|
str: str,
|
||||||
|
ipnet: net,
|
||||||
|
allowed: allowed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
f.mut.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
//check if plain ip
|
||||||
|
if ip := net.ParseIP(str); ip != nil {
|
||||||
|
f.mut.Lock()
|
||||||
|
f.ips[ip.String()] = allowed
|
||||||
|
f.mut.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
//ToggleDefault alters the default setting
|
||||||
|
func (f *ipFilter) ToggleDefault(allowed bool) {
|
||||||
|
f.mut.Lock()
|
||||||
|
f.defaultAllowed = allowed
|
||||||
|
f.mut.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
//Allowed returns if a given IP can pass through the filter
|
||||||
|
func (f *ipFilter) Allowed(ipstr string) bool {
|
||||||
|
return f.NetAllowed(net.ParseIP(ipstr))
|
||||||
|
}
|
||||||
|
|
||||||
|
//NetAllowed returns if a given net.IP can pass through the filter
|
||||||
|
func (f *ipFilter) NetAllowed(ip net.IP) bool {
|
||||||
|
//invalid ip
|
||||||
|
if ip == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
//read lock entire function
|
||||||
|
//except for db access
|
||||||
|
f.mut.RLock()
|
||||||
|
defer f.mut.RUnlock()
|
||||||
|
//check single ips
|
||||||
|
allowed, ok := f.ips[ip.String()]
|
||||||
|
if ok {
|
||||||
|
return allowed
|
||||||
|
}
|
||||||
|
//scan subnets for any allow/block
|
||||||
|
blocked := false
|
||||||
|
for _, subnet := range f.subnets {
|
||||||
|
if subnet.ipnet.Contains(ip) {
|
||||||
|
if subnet.allowed {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
blocked = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if blocked {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
//use default setting
|
||||||
|
return f.defaultAllowed
|
||||||
|
}
|
||||||
|
|
||||||
|
//Blocked returns if a given IP can NOT pass through the filter
|
||||||
|
func (f *ipFilter) Blocked(ip string) bool {
|
||||||
|
return !f.Allowed(ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
//NetBlocked returns if a given net.IP can NOT pass through the filter
|
||||||
|
func (f *ipFilter) NetBlocked(ip net.IP) bool {
|
||||||
|
return !f.NetAllowed(ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
//WrapIPFilter the provided handler with simple IP blocking middleware
|
||||||
|
//using this IP filter and its configuration
|
||||||
|
func (f *ipFilter) Wrap(next http.Handler) http.Handler {
|
||||||
|
return &ipFilterMiddleware{ipFilter: f, next: next}
|
||||||
|
}
|
||||||
|
|
||||||
|
//WrapIPFilter is equivalent to newIPFilter(opts) then Wrap(next)
|
||||||
|
func WrapIPFilter(next http.Handler, opts IPFilterOptions) http.Handler {
|
||||||
|
return newIPFilter(opts).Wrap(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ipFilterMiddleware struct {
|
||||||
|
*ipFilter
|
||||||
|
next http.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ipFilterMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
remoteIP := realip.FromRequest(r)
|
||||||
|
|
||||||
|
if !m.ipFilter.Allowed(remoteIP) {
|
||||||
|
//show simple forbidden text
|
||||||
|
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
//success!
|
||||||
|
m.next.ServeHTTP(w, r)
|
||||||
|
}
|
||||||
255
server/server.go
255
server/server.go
@ -25,8 +25,10 @@ THE SOFTWARE.
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
crypto_rand "crypto/rand"
|
||||||
|
"encoding/binary"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
gorillaHandlers "github.com/gorilla/handlers"
|
||||||
"log"
|
"log"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"mime"
|
"mime"
|
||||||
@ -46,6 +48,7 @@ import (
|
|||||||
"github.com/VojtechVitek/ratelimit/memory"
|
"github.com/VojtechVitek/ratelimit/memory"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
|
|
||||||
|
// import pprof
|
||||||
_ "net/http/pprof"
|
_ "net/http/pprof"
|
||||||
|
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
@ -57,25 +60,37 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
const SERVER_INFO = "transfer.sh"
|
|
||||||
|
|
||||||
// parse request with maximum memory of _24Kilobits
|
// parse request with maximum memory of _24Kilobits
|
||||||
const _24K = (1 << 10) * 24
|
const _24K = (1 << 3) * 24
|
||||||
|
|
||||||
|
// parse request with maximum memory of _5Megabytes
|
||||||
|
const _5M = (1 << 20) * 5
|
||||||
|
|
||||||
|
// OptionFn is the option function type
|
||||||
type OptionFn func(*Server)
|
type OptionFn func(*Server)
|
||||||
|
|
||||||
|
// ClamavHost sets clamav host
|
||||||
func ClamavHost(s string) OptionFn {
|
func ClamavHost(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.ClamAVDaemonHost = s
|
srvr.ClamAVDaemonHost = s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PerformClamavPrescan enables clamav prescan on upload
|
||||||
|
func PerformClamavPrescan(b bool) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.performClamavPrescan = b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// VirustotalKey sets virus total key
|
||||||
func VirustotalKey(s string) OptionFn {
|
func VirustotalKey(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.VirusTotalKey = s
|
srvr.VirusTotalKey = s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Listener set listener
|
||||||
func Listener(s string) OptionFn {
|
func Listener(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.ListenerString = s
|
srvr.ListenerString = s
|
||||||
@ -83,18 +98,36 @@ func Listener(s string) OptionFn {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CorsDomains sets CORS domains
|
||||||
|
func CorsDomains(s string) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.CorsDomains = s
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// EmailContact sets email contact
|
||||||
|
func EmailContact(emailContact string) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.emailContact = emailContact
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoogleAnalytics sets GA key
|
||||||
func GoogleAnalytics(gaKey string) OptionFn {
|
func GoogleAnalytics(gaKey string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.gaKey = gaKey
|
srvr.gaKey = gaKey
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UserVoice sets UV key
|
||||||
func UserVoice(userVoiceKey string) OptionFn {
|
func UserVoice(userVoiceKey string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.userVoiceKey = userVoiceKey
|
srvr.userVoiceKey = userVoiceKey
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TLSListener sets TLS listener and option
|
||||||
func TLSListener(s string, t bool) OptionFn {
|
func TLSListener(s string, t bool) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.TLSListenerString = s
|
srvr.TLSListenerString = s
|
||||||
@ -103,12 +136,14 @@ func TLSListener(s string, t bool) OptionFn {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProfileListener sets profile listener
|
||||||
func ProfileListener(s string) OptionFn {
|
func ProfileListener(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.ProfileListenerString = s
|
srvr.ProfileListenerString = s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WebPath sets web path
|
||||||
func WebPath(s string) OptionFn {
|
func WebPath(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
if s[len(s)-1:] != "/" {
|
if s[len(s)-1:] != "/" {
|
||||||
@ -119,6 +154,25 @@ func WebPath(s string) OptionFn {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProxyPath sets proxy path
|
||||||
|
func ProxyPath(s string) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
if s[len(s)-1:] != "/" {
|
||||||
|
s = s + string(filepath.Separator)
|
||||||
|
}
|
||||||
|
|
||||||
|
srvr.proxyPath = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProxyPort sets proxy port
|
||||||
|
func ProxyPort(s string) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.proxyPort = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TempPath sets temp path
|
||||||
func TempPath(s string) OptionFn {
|
func TempPath(s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
if s[len(s)-1:] != "/" {
|
if s[len(s)-1:] != "/" {
|
||||||
@ -129,41 +183,78 @@ func TempPath(s string) OptionFn {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func LogFile(s string) OptionFn {
|
// LogFile sets log file
|
||||||
|
func LogFile(logger *log.Logger, s string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
f, err := os.OpenFile(s, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
|
f, err := os.OpenFile(s, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("error opening file: %v", err)
|
logger.Fatalf("error opening file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.SetOutput(f)
|
logger.SetOutput(f)
|
||||||
|
srvr.logger = logger
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Logger sets logger
|
||||||
|
func Logger(logger *log.Logger) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.logger = logger
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxUploadSize sets max upload size
|
||||||
|
func MaxUploadSize(kbytes int64) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.maxUploadSize = kbytes * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// RateLimit set rate limit
|
||||||
func RateLimit(requests int) OptionFn {
|
func RateLimit(requests int) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.rateLimitRequests = requests
|
srvr.rateLimitRequests = requests
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func ForceHTTPs() OptionFn {
|
// RandomTokenLength sets random token length
|
||||||
|
func RandomTokenLength(length int) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.forceHTTPs = true
|
srvr.randomTokenLength = length
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Purge sets purge days and option
|
||||||
|
func Purge(days, interval int) OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.purgeDays = time.Duration(days) * time.Hour * 24
|
||||||
|
srvr.purgeInterval = time.Duration(interval) * time.Hour
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForceHTTPS sets forcing https
|
||||||
|
func ForceHTTPS() OptionFn {
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.forceHTTPS = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnableProfiler sets enable profiler
|
||||||
func EnableProfiler() OptionFn {
|
func EnableProfiler() OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.profilerEnabled = true
|
srvr.profilerEnabled = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UseStorage set storage to use
|
||||||
func UseStorage(s Storage) OptionFn {
|
func UseStorage(s Storage) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.storage = s
|
srvr.storage = s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UseLetsEncrypt set letsencrypt usage
|
||||||
func UseLetsEncrypt(hosts []string) OptionFn {
|
func UseLetsEncrypt(hosts []string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
cacheDir := "./cache/"
|
cacheDir := "./cache/"
|
||||||
@ -192,6 +283,7 @@ func UseLetsEncrypt(hosts []string) OptionFn {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TLSConfig sets TLS config
|
||||||
func TLSConfig(cert, pk string) OptionFn {
|
func TLSConfig(cert, pk string) OptionFn {
|
||||||
certificate, err := tls.LoadX509KeyPair(cert, pk)
|
certificate, err := tls.LoadX509KeyPair(cert, pk)
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
@ -203,40 +295,72 @@ func TLSConfig(cert, pk string) OptionFn {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func HttpAuthCredentials(user string, pass string) OptionFn {
|
// HTTPAuthCredentials sets basic http auth credentials
|
||||||
|
func HTTPAuthCredentials(user string, pass string) OptionFn {
|
||||||
return func(srvr *Server) {
|
return func(srvr *Server) {
|
||||||
srvr.AuthUser = user
|
srvr.AuthUser = user
|
||||||
srvr.AuthPass = pass
|
srvr.AuthPass = pass
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FilterOptions sets ip filtering
|
||||||
|
func FilterOptions(options IPFilterOptions) OptionFn {
|
||||||
|
for i, allowedIP := range options.AllowedIPs {
|
||||||
|
options.AllowedIPs[i] = strings.TrimSpace(allowedIP)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, blockedIP := range options.BlockedIPs {
|
||||||
|
options.BlockedIPs[i] = strings.TrimSpace(blockedIP)
|
||||||
|
}
|
||||||
|
|
||||||
|
return func(srvr *Server) {
|
||||||
|
srvr.ipFilterOptions = &options
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server is the main application
|
||||||
type Server struct {
|
type Server struct {
|
||||||
AuthUser string
|
AuthUser string
|
||||||
AuthPass string
|
AuthPass string
|
||||||
|
|
||||||
|
logger *log.Logger
|
||||||
|
|
||||||
tlsConfig *tls.Config
|
tlsConfig *tls.Config
|
||||||
|
|
||||||
profilerEnabled bool
|
profilerEnabled bool
|
||||||
|
|
||||||
locks map[string]*sync.Mutex
|
locks sync.Map
|
||||||
|
|
||||||
|
maxUploadSize int64
|
||||||
rateLimitRequests int
|
rateLimitRequests int
|
||||||
|
|
||||||
|
purgeDays time.Duration
|
||||||
|
purgeInterval time.Duration
|
||||||
|
|
||||||
storage Storage
|
storage Storage
|
||||||
|
|
||||||
forceHTTPs bool
|
forceHTTPS bool
|
||||||
|
|
||||||
VirusTotalKey string
|
randomTokenLength int
|
||||||
ClamAVDaemonHost string
|
|
||||||
|
ipFilterOptions *IPFilterOptions
|
||||||
|
|
||||||
|
VirusTotalKey string
|
||||||
|
ClamAVDaemonHost string
|
||||||
|
performClamavPrescan bool
|
||||||
|
|
||||||
tempPath string
|
tempPath string
|
||||||
|
|
||||||
webPath string
|
webPath string
|
||||||
|
proxyPath string
|
||||||
|
proxyPort string
|
||||||
|
emailContact string
|
||||||
gaKey string
|
gaKey string
|
||||||
userVoiceKey string
|
userVoiceKey string
|
||||||
|
|
||||||
TLSListenerOnly bool
|
TLSListenerOnly bool
|
||||||
|
|
||||||
|
CorsDomains string
|
||||||
ListenerString string
|
ListenerString string
|
||||||
TLSListenerString string
|
TLSListenerString string
|
||||||
ProfileListenerString string
|
ProfileListenerString string
|
||||||
@ -246,9 +370,10 @@ type Server struct {
|
|||||||
LetsEncryptCache string
|
LetsEncryptCache string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// New is the factory fot Server
|
||||||
func New(options ...OptionFn) (*Server, error) {
|
func New(options ...OptionFn) (*Server, error) {
|
||||||
s := &Server{
|
s := &Server{
|
||||||
locks: map[string]*sync.Mutex{},
|
locks: sync.Map{},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, optionFn := range options {
|
for _, optionFn := range options {
|
||||||
@ -259,9 +384,14 @@ func New(options ...OptionFn) (*Server, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rand.Seed(time.Now().UTC().UnixNano())
|
var seedBytes [8]byte
|
||||||
|
if _, err := crypto_rand.Read(seedBytes[:]); err != nil {
|
||||||
|
panic("cannot obtain cryptographically secure seed")
|
||||||
|
}
|
||||||
|
rand.Seed(int64(binary.LittleEndian.Uint64(seedBytes[:])))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run starts Server
|
||||||
func (s *Server) Run() {
|
func (s *Server) Run() {
|
||||||
listening := false
|
listening := false
|
||||||
|
|
||||||
@ -269,9 +399,9 @@ func (s *Server) Run() {
|
|||||||
listening = true
|
listening = true
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
fmt.Println("Profiled listening at: :6060")
|
s.logger.Println("Profiled listening at: :6060")
|
||||||
|
|
||||||
http.ListenAndServe(":6060", nil)
|
_ = http.ListenAndServe(":6060", nil)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -280,7 +410,7 @@ func (s *Server) Run() {
|
|||||||
var fs http.FileSystem
|
var fs http.FileSystem
|
||||||
|
|
||||||
if s.webPath != "" {
|
if s.webPath != "" {
|
||||||
log.Println("Using static file path: ", s.webPath)
|
s.logger.Println("Using static file path: ", s.webPath)
|
||||||
|
|
||||||
fs = http.Dir(s.webPath)
|
fs = http.Dir(s.webPath)
|
||||||
|
|
||||||
@ -299,23 +429,35 @@ func (s *Server) Run() {
|
|||||||
for _, path := range web.AssetNames() {
|
for _, path := range web.AssetNames() {
|
||||||
bytes, err := web.Asset(path)
|
bytes, err := web.Asset(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Panicf("Unable to parse: path=%s, err=%s", path, err)
|
s.logger.Panicf("Unable to parse: path=%s, err=%s", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
if strings.HasSuffix(path, ".html") {
|
||||||
textTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Println("Unable to parse html template", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(path, ".txt") {
|
||||||
|
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Println("Unable to parse text template", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
staticHandler := http.FileServer(fs)
|
staticHandler := http.FileServer(fs)
|
||||||
|
|
||||||
r.PathPrefix("/images/").Handler(staticHandler)
|
r.PathPrefix("/images/").Handler(staticHandler).Methods("GET")
|
||||||
r.PathPrefix("/styles/").Handler(staticHandler)
|
r.PathPrefix("/styles/").Handler(staticHandler).Methods("GET")
|
||||||
r.PathPrefix("/scripts/").Handler(staticHandler)
|
r.PathPrefix("/scripts/").Handler(staticHandler).Methods("GET")
|
||||||
r.PathPrefix("/fonts/").Handler(staticHandler)
|
r.PathPrefix("/fonts/").Handler(staticHandler).Methods("GET")
|
||||||
r.PathPrefix("/ico/").Handler(staticHandler)
|
r.PathPrefix("/ico/").Handler(staticHandler).Methods("GET")
|
||||||
r.PathPrefix("/favicon.ico").Handler(staticHandler)
|
r.HandleFunc("/favicon.ico", staticHandler.ServeHTTP).Methods("GET")
|
||||||
r.PathPrefix("/robots.txt").Handler(staticHandler)
|
r.HandleFunc("/robots.txt", staticHandler.ServeHTTP).Methods("GET")
|
||||||
|
|
||||||
|
r.HandleFunc("/{filename:(?:favicon\\.ico|robots\\.txt|health\\.html)}", s.basicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
||||||
|
|
||||||
r.HandleFunc("/health.html", healthHandler).Methods("GET")
|
r.HandleFunc("/health.html", healthHandler).Methods("GET")
|
||||||
r.HandleFunc("/", s.viewHandler).Methods("GET")
|
r.HandleFunc("/", s.viewHandler).Methods("GET")
|
||||||
@ -341,7 +483,7 @@ func (s *Server) Run() {
|
|||||||
|
|
||||||
u, err := url.Parse(r.Referer())
|
u, err := url.Parse(r.Referer())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
s.logger.Fatal(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -359,21 +501,44 @@ func (s *Server) Run() {
|
|||||||
|
|
||||||
r.HandleFunc("/{filename}/virustotal", s.virusTotalHandler).Methods("PUT")
|
r.HandleFunc("/{filename}/virustotal", s.virusTotalHandler).Methods("PUT")
|
||||||
r.HandleFunc("/{filename}/scan", s.scanHandler).Methods("PUT")
|
r.HandleFunc("/{filename}/scan", s.scanHandler).Methods("PUT")
|
||||||
r.HandleFunc("/put/{filename}", s.BasicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
r.HandleFunc("/put/{filename}", s.basicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
||||||
r.HandleFunc("/upload/{filename}", s.BasicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
r.HandleFunc("/upload/{filename}", s.basicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
||||||
r.HandleFunc("/{filename}", s.BasicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
r.HandleFunc("/{filename}", s.basicAuthHandler(http.HandlerFunc(s.putHandler))).Methods("PUT")
|
||||||
r.HandleFunc("/", s.BasicAuthHandler(http.HandlerFunc(s.postHandler))).Methods("POST")
|
r.HandleFunc("/", s.basicAuthHandler(http.HandlerFunc(s.postHandler))).Methods("POST")
|
||||||
// r.HandleFunc("/{page}", viewHandler).Methods("GET")
|
// r.HandleFunc("/{page}", viewHandler).Methods("GET")
|
||||||
|
|
||||||
r.HandleFunc("/{token}/{filename}/{deletionToken}", s.deleteHandler).Methods("DELETE")
|
r.HandleFunc("/{token}/{filename}/{deletionToken}", s.deleteHandler).Methods("DELETE")
|
||||||
|
|
||||||
r.NotFoundHandler = http.HandlerFunc(s.notFoundHandler)
|
r.NotFoundHandler = http.HandlerFunc(s.notFoundHandler)
|
||||||
|
|
||||||
mime.AddExtensionType(".md", "text/x-markdown")
|
_ = mime.AddExtensionType(".md", "text/x-markdown")
|
||||||
|
|
||||||
log.Printf("Transfer.sh server started.\nusing temp folder: %s\nusing storage provider: %s", s.tempPath, s.storage.Type())
|
s.logger.Printf("Transfer.sh server started.\nusing temp folder: %s\nusing storage provider: %s", s.tempPath, s.storage.Type())
|
||||||
|
|
||||||
h := handlers.PanicHandler(handlers.LogHandler(LoveHandler(s.RedirectHandler(r)), handlers.NewLogOptions(log.Printf, "_default_")), nil)
|
var cors func(http.Handler) http.Handler
|
||||||
|
if len(s.CorsDomains) > 0 {
|
||||||
|
cors = gorillaHandlers.CORS(
|
||||||
|
gorillaHandlers.AllowedHeaders([]string{"*"}),
|
||||||
|
gorillaHandlers.AllowedOrigins(strings.Split(s.CorsDomains, ",")),
|
||||||
|
gorillaHandlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"}),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
cors = func(h http.Handler) http.Handler {
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h := handlers.PanicHandler(
|
||||||
|
ipFilterHandler(
|
||||||
|
handlers.LogHandler(
|
||||||
|
LoveHandler(
|
||||||
|
s.RedirectHandler(cors(r))),
|
||||||
|
handlers.NewLogOptions(s.logger.Printf, "_default_"),
|
||||||
|
),
|
||||||
|
s.ipFilterOptions,
|
||||||
|
),
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
if !s.TLSListenerOnly {
|
if !s.TLSListenerOnly {
|
||||||
srvr := &http.Server{
|
srvr := &http.Server{
|
||||||
@ -382,16 +547,16 @@ func (s *Server) Run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
listening = true
|
listening = true
|
||||||
log.Printf("listening on port: %v\n", s.ListenerString)
|
s.logger.Printf("listening on port: %v\n", s.ListenerString)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
srvr.ListenAndServe()
|
_ = srvr.ListenAndServe()
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.TLSListenerString != "" {
|
if s.TLSListenerString != "" {
|
||||||
listening = true
|
listening = true
|
||||||
log.Printf("listening on port: %v\n", s.TLSListenerString)
|
s.logger.Printf("listening on port: %v\n", s.TLSListenerString)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
s := &http.Server{
|
s := &http.Server{
|
||||||
@ -406,7 +571,11 @@ func (s *Server) Run() {
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("---------------------------")
|
s.logger.Printf("---------------------------")
|
||||||
|
|
||||||
|
if s.purgeDays > 0 {
|
||||||
|
go s.purgeHandler()
|
||||||
|
}
|
||||||
|
|
||||||
term := make(chan os.Signal, 1)
|
term := make(chan os.Signal, 1)
|
||||||
signal.Notify(term, os.Interrupt)
|
signal.Notify(term, os.Interrupt)
|
||||||
@ -415,8 +584,8 @@ func (s *Server) Run() {
|
|||||||
if listening {
|
if listening {
|
||||||
<-term
|
<-term
|
||||||
} else {
|
} else {
|
||||||
log.Printf("No listener active.")
|
s.logger.Printf("No listener active.")
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Server stopped.")
|
s.logger.Printf("Server stopped.")
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@ -1,4 +1,7 @@
|
|||||||
Copyright (c) 2014 Tom Harwood
|
/*
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2020- Andrea Spacca and Stefan Benten.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
@ -17,3 +20,26 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
THE SOFTWARE.
|
THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math/rand"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SYMBOLS characters used for short-urls
|
||||||
|
SYMBOLS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||||
|
)
|
||||||
|
|
||||||
|
// generate a token
|
||||||
|
func token(length int) string {
|
||||||
|
result := ""
|
||||||
|
for i := 0; i < length; i++ {
|
||||||
|
x := rand.Intn(len(SYMBOLS) - 1)
|
||||||
|
result = string(SYMBOLS[x]) + result
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
15
server/token_test.go
Normal file
15
server/token_test.go
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func BenchmarkTokenConcat(b *testing.B) {
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = token(5) + token(5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkTokenLonger(b *testing.B) {
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = token(10)
|
||||||
|
}
|
||||||
|
}
|
||||||
102
server/utils.go
102
server/utils.go
@ -2,6 +2,8 @@
|
|||||||
The MIT License (MIT)
|
The MIT License (MIT)
|
||||||
|
|
||||||
Copyright (c) 2014-2017 DutchCoders [https://github.com/dutchcoders/]
|
Copyright (c) 2014-2017 DutchCoders [https://github.com/dutchcoders/]
|
||||||
|
Copyright (c) 2018-2020 Andrea Spacca.
|
||||||
|
Copyright (c) 2020- Andrea Spacca and Stefan Benten.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
@ -25,61 +27,31 @@ THE SOFTWARE.
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/mail"
|
"net/mail"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/goamz/goamz/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/goamz/goamz/s3"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
"github.com/golang/gddo/httputil/header"
|
"github.com/golang/gddo/httputil/header"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getBucket(accessKey, secretKey, bucket, endpoint string) (*s3.Bucket, error) {
|
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session {
|
||||||
auth, err := aws.GetAuth(accessKey, secretKey, "", time.Time{})
|
return session.Must(session.NewSession(&aws.Config{
|
||||||
if err != nil {
|
Region: aws.String(region),
|
||||||
return nil, err
|
Endpoint: aws.String(endpoint),
|
||||||
}
|
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||||
|
S3ForcePathStyle: aws.Bool(forcePathStyle),
|
||||||
var EUWestWithoutHTTPS = aws.Region{
|
}))
|
||||||
Name: "eu-west-1",
|
|
||||||
EC2Endpoint: "https://ec2.eu-west-1.amazonaws.com",
|
|
||||||
S3Endpoint: endpoint,
|
|
||||||
S3BucketEndpoint: "",
|
|
||||||
S3LocationConstraint: true,
|
|
||||||
S3LowercaseBucket: true,
|
|
||||||
SDBEndpoint: "https://sdb.eu-west-1.amazonaws.com",
|
|
||||||
SESEndpoint: "https://email.eu-west-1.amazonaws.com",
|
|
||||||
SNSEndpoint: "https://sns.eu-west-1.amazonaws.com",
|
|
||||||
SQSEndpoint: "https://sqs.eu-west-1.amazonaws.com",
|
|
||||||
IAMEndpoint: "https://iam.amazonaws.com",
|
|
||||||
ELBEndpoint: "https://elasticloadbalancing.eu-west-1.amazonaws.com",
|
|
||||||
DynamoDBEndpoint: "https://dynamodb.eu-west-1.amazonaws.com",
|
|
||||||
CloudWatchServicepoint: aws.ServiceInfo{
|
|
||||||
Endpoint: "https://monitoring.eu-west-1.amazonaws.com",
|
|
||||||
Signer: aws.V2Signature,
|
|
||||||
},
|
|
||||||
AutoScalingEndpoint: "https://autoscaling.eu-west-1.amazonaws.com",
|
|
||||||
RDSEndpoint: aws.ServiceInfo{
|
|
||||||
Endpoint: "https://rds.eu-west-1.amazonaws.com",
|
|
||||||
Signer: aws.V2Signature,
|
|
||||||
},
|
|
||||||
STSEndpoint: "https://sts.amazonaws.com",
|
|
||||||
CloudFormationEndpoint: "https://cloudformation.eu-west-1.amazonaws.com",
|
|
||||||
ECSEndpoint: "https://ecs.eu-west-1.amazonaws.com",
|
|
||||||
DynamoDBStreamsEndpoint: "https://streams.dynamodb.eu-west-1.amazonaws.com",
|
|
||||||
}
|
|
||||||
|
|
||||||
conn := s3.New(auth, EUWestWithoutHTTPS)
|
|
||||||
b := conn.Bucket(bucket)
|
|
||||||
return b, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func formatNumber(format string, s uint64) string {
|
func formatNumber(format string, s uint64) string {
|
||||||
|
return renderFloat(format, float64(s))
|
||||||
return RenderFloat(format, float64(s))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var renderFloatPrecisionMultipliers = [10]float64{
|
var renderFloatPrecisionMultipliers = [10]float64{
|
||||||
@ -108,7 +80,7 @@ var renderFloatPrecisionRounders = [10]float64{
|
|||||||
0.0000000005,
|
0.0000000005,
|
||||||
}
|
}
|
||||||
|
|
||||||
func RenderFloat(format string, n float64) string {
|
func renderFloat(format string, n float64) string {
|
||||||
// Special cases:
|
// Special cases:
|
||||||
// NaN = "NaN"
|
// NaN = "NaN"
|
||||||
// +Inf = "+Infinity"
|
// +Inf = "+Infinity"
|
||||||
@ -156,7 +128,7 @@ func RenderFloat(format string, n float64) string {
|
|||||||
// +0000
|
// +0000
|
||||||
if formatDirectiveIndices[0] == 0 {
|
if formatDirectiveIndices[0] == 0 {
|
||||||
if formatDirectiveChars[formatDirectiveIndices[0]] != '+' {
|
if formatDirectiveChars[formatDirectiveIndices[0]] != '+' {
|
||||||
panic("RenderFloat(): invalid positive sign directive")
|
panic("renderFloat(): invalid positive sign directive")
|
||||||
}
|
}
|
||||||
positiveStr = "+"
|
positiveStr = "+"
|
||||||
formatDirectiveIndices = formatDirectiveIndices[1:]
|
formatDirectiveIndices = formatDirectiveIndices[1:]
|
||||||
@ -170,7 +142,7 @@ func RenderFloat(format string, n float64) string {
|
|||||||
// 000,000.00
|
// 000,000.00
|
||||||
if len(formatDirectiveIndices) == 2 {
|
if len(formatDirectiveIndices) == 2 {
|
||||||
if (formatDirectiveIndices[1] - formatDirectiveIndices[0]) != 4 {
|
if (formatDirectiveIndices[1] - formatDirectiveIndices[0]) != 4 {
|
||||||
panic("RenderFloat(): thousands separator directive must be followed by 3 digit-specifiers")
|
panic("renderFloat(): thousands separator directive must be followed by 3 digit-specifiers")
|
||||||
}
|
}
|
||||||
thousandStr = string(formatDirectiveChars[formatDirectiveIndices[0]])
|
thousandStr = string(formatDirectiveChars[formatDirectiveIndices[0]])
|
||||||
formatDirectiveIndices = formatDirectiveIndices[1:]
|
formatDirectiveIndices = formatDirectiveIndices[1:]
|
||||||
@ -230,8 +202,8 @@ func RenderFloat(format string, n float64) string {
|
|||||||
return signStr + intStr + decimalStr + fracStr
|
return signStr + intStr + decimalStr + fracStr
|
||||||
}
|
}
|
||||||
|
|
||||||
func RenderInteger(format string, n int) string {
|
func renderInteger(format string, n int) string {
|
||||||
return RenderFloat(format, float64(n))
|
return renderFloat(format, float64(n))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request.RemoteAddress contains port, which we want to remove i.e.:
|
// Request.RemoteAddress contains port, which we want to remove i.e.:
|
||||||
@ -284,3 +256,37 @@ func acceptsHTML(hdr http.Header) bool {
|
|||||||
|
|
||||||
return (false)
|
return (false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func formatSize(size int64) string {
|
||||||
|
sizeFloat := float64(size)
|
||||||
|
base := math.Log(sizeFloat) / math.Log(1024)
|
||||||
|
|
||||||
|
sizeOn := math.Pow(1024, base-math.Floor(base))
|
||||||
|
|
||||||
|
var round float64
|
||||||
|
pow := math.Pow(10, float64(2))
|
||||||
|
digit := pow * sizeOn
|
||||||
|
round = math.Floor(digit)
|
||||||
|
|
||||||
|
newVal := round / pow
|
||||||
|
|
||||||
|
var suffixes [5]string
|
||||||
|
suffixes[0] = "B"
|
||||||
|
suffixes[1] = "KB"
|
||||||
|
suffixes[2] = "MB"
|
||||||
|
suffixes[3] = "GB"
|
||||||
|
suffixes[4] = "TB"
|
||||||
|
|
||||||
|
getSuffix := suffixes[int(math.Floor(base))]
|
||||||
|
return fmt.Sprintf("%s %s", strconv.FormatFloat(newVal, 'f', -1, 64), getSuffix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeClose(c io.Closer) {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.Close(); err != nil {
|
||||||
|
fmt.Println("Received close error:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -26,11 +26,8 @@ package server
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
_ "github.com/PuerkitoBio/ghost/handlers"
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
|
|
||||||
virustotal "github.com/dutchcoders/go-virustotal"
|
virustotal "github.com/dutchcoders/go-virustotal"
|
||||||
@ -44,22 +41,20 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
contentLength := r.ContentLength
|
contentLength := r.ContentLength
|
||||||
contentType := r.Header.Get("Content-Type")
|
contentType := r.Header.Get("Content-Type")
|
||||||
|
|
||||||
log.Printf("Submitting to VirusTotal: %s %d %s", filename, contentLength, contentType)
|
s.logger.Printf("Submitting to VirusTotal: %s %d %s", filename, contentLength, contentType)
|
||||||
|
|
||||||
vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
|
vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 500)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
||||||
var reader io.Reader
|
reader := r.Body
|
||||||
|
|
||||||
reader = r.Body
|
|
||||||
|
|
||||||
result, err := vt.Scan(filename, reader)
|
result, err := vt.Scan(filename, reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 500)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Println(result)
|
s.logger.Println(result)
|
||||||
w.Write([]byte(fmt.Sprintf("%v\n", result.Permalink)))
|
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", result.Permalink)))
|
||||||
}
|
}
|
||||||
|
|||||||
23
vendor/cloud.google.com/go/.travis.yml
generated
vendored
23
vendor/cloud.google.com/go/.travis.yml
generated
vendored
@ -1,23 +0,0 @@
|
|||||||
sudo: false
|
|
||||||
language: go
|
|
||||||
go:
|
|
||||||
- 1.6.x
|
|
||||||
- 1.7.x
|
|
||||||
- 1.8.x
|
|
||||||
- 1.9.x
|
|
||||||
- 1.10.x
|
|
||||||
install:
|
|
||||||
- go get -v cloud.google.com/go/...
|
|
||||||
script:
|
|
||||||
- openssl aes-256-cbc -K $encrypted_a8b3f4fc85f4_key -iv $encrypted_a8b3f4fc85f4_iv -in keys.tar.enc -out keys.tar -d
|
|
||||||
- tar xvf keys.tar
|
|
||||||
- GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762"
|
|
||||||
GCLOUD_TESTS_GOLANG_KEY="$(pwd)/dulcet-port-762-key.json"
|
|
||||||
GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID="gcloud-golang-firestore-tests"
|
|
||||||
GCLOUD_TESTS_GOLANG_FIRESTORE_KEY="$(pwd)/gcloud-golang-firestore-tests-key.json"
|
|
||||||
GCLOUD_TESTS_GOLANG_KEYRING="projects/dulcet-port-762/locations/us/keyRings/go-integration-test"
|
|
||||||
./run-tests.sh $TRAVIS_COMMIT
|
|
||||||
env:
|
|
||||||
matrix:
|
|
||||||
# The GCLOUD_TESTS_API_KEY environment variable.
|
|
||||||
secure: VdldogUOoubQ60LhuHJ+g/aJoBiujkSkWEWl79Zb8cvQorcQbxISS+JsOOp4QkUOU4WwaHAm8/3pIH1QMWOR6O78DaLmDKi5Q4RpkVdCpUXy+OAfQaZIcBsispMrjxLXnqFjo9ELnrArfjoeCTzaX0QTCfwQwVmigC8rR30JBKI=
|
|
||||||
15
vendor/cloud.google.com/go/AUTHORS
generated
vendored
15
vendor/cloud.google.com/go/AUTHORS
generated
vendored
@ -1,15 +0,0 @@
|
|||||||
# This is the official list of cloud authors for copyright purposes.
|
|
||||||
# This file is distinct from the CONTRIBUTORS files.
|
|
||||||
# See the latter for an explanation.
|
|
||||||
|
|
||||||
# Names should be added to this file as:
|
|
||||||
# Name or Organization <email address>
|
|
||||||
# The email address is not required for organizations.
|
|
||||||
|
|
||||||
Filippo Valsorda <hi@filippo.io>
|
|
||||||
Google Inc.
|
|
||||||
Ingo Oeser <nightlyone@googlemail.com>
|
|
||||||
Palm Stone Games, Inc.
|
|
||||||
Paweł Knap <pawelknap88@gmail.com>
|
|
||||||
Péter Szilágyi <peterke@gmail.com>
|
|
||||||
Tyler Treat <ttreat31@gmail.com>
|
|
||||||
167
vendor/cloud.google.com/go/CONTRIBUTING.md
generated
vendored
167
vendor/cloud.google.com/go/CONTRIBUTING.md
generated
vendored
@ -1,167 +0,0 @@
|
|||||||
# Contributing
|
|
||||||
|
|
||||||
1. Sign one of the contributor license agreements below.
|
|
||||||
1. `go get golang.org/x/review/git-codereview` to install the code reviewing tool.
|
|
||||||
1. You will need to ensure that your `GOBIN` directory (by default
|
|
||||||
`$GOPATH/bin`) is in your `PATH` so that git can find the command.
|
|
||||||
1. If you would like, you may want to set up aliases for git-codereview,
|
|
||||||
such that `git codereview change` becomes `git change`. See the
|
|
||||||
[godoc](https://godoc.org/golang.org/x/review/git-codereview) for details.
|
|
||||||
1. Should you run into issues with the git-codereview tool, please note
|
|
||||||
that all error messages will assume that you have set up these
|
|
||||||
aliases.
|
|
||||||
1. Get the cloud package by running `go get -d cloud.google.com/go`.
|
|
||||||
1. If you have already checked out the source, make sure that the remote git
|
|
||||||
origin is https://code.googlesource.com/gocloud:
|
|
||||||
|
|
||||||
git remote set-url origin https://code.googlesource.com/gocloud
|
|
||||||
1. Make sure your auth is configured correctly by visiting
|
|
||||||
https://code.googlesource.com, clicking "Generate Password", and following
|
|
||||||
the directions.
|
|
||||||
1. Make changes and create a change by running `git codereview change <name>`,
|
|
||||||
provide a commit message, and use `git codereview mail` to create a Gerrit CL.
|
|
||||||
1. Keep amending to the change with `git codereview change` and mail as your receive
|
|
||||||
feedback. Each new mailed amendment will create a new patch set for your change in Gerrit.
|
|
||||||
|
|
||||||
## Integration Tests
|
|
||||||
|
|
||||||
In addition to the unit tests, you may run the integration test suite.
|
|
||||||
|
|
||||||
To run the integrations tests, creating and configuration of a project in the
|
|
||||||
Google Developers Console is required.
|
|
||||||
|
|
||||||
After creating a project, you must [create a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount).
|
|
||||||
Ensure the project-level **Owner**
|
|
||||||
[IAM role](console.cloud.google.com/iam-admin/iam/project) role is added to the
|
|
||||||
service account. Alternatively, the account can be granted all of the following roles:
|
|
||||||
- **Editor**
|
|
||||||
- **Logs Configuration Writer**
|
|
||||||
- **PubSub Admin**
|
|
||||||
|
|
||||||
Once you create a project, set the following environment variables to be able to
|
|
||||||
run the against the actual APIs.
|
|
||||||
|
|
||||||
- **GCLOUD_TESTS_GOLANG_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455)
|
|
||||||
- **GCLOUD_TESTS_GOLANG_KEY**: The path to the JSON key file.
|
|
||||||
|
|
||||||
Some packages require additional environment variables to be set:
|
|
||||||
|
|
||||||
- firestore
|
|
||||||
- **GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID**: project ID for Firestore.
|
|
||||||
- **GCLOUD_TESTS_GOLANG_FIRESTORE_KEY**: The path to the JSON key file.
|
|
||||||
- storage
|
|
||||||
- **GCLOUD_TESTS_GOLANG_KEYRING**: The full name of the keyring for the tests, in the
|
|
||||||
form "projects/P/locations/L/keyRings/R".
|
|
||||||
- translate
|
|
||||||
- **GCLOUD_TESTS_API_KEY**: API key for using the Translate API.
|
|
||||||
- profiler
|
|
||||||
- **GCLOUD_TESTS_GOLANG_ZONE**: Compute Engine zone.
|
|
||||||
|
|
||||||
Install the [gcloud command-line tool][gcloudcli] to your machine and use it
|
|
||||||
to create some resources used in integration tests.
|
|
||||||
|
|
||||||
From the project's root directory:
|
|
||||||
|
|
||||||
``` sh
|
|
||||||
# Set the default project in your env.
|
|
||||||
$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID
|
|
||||||
|
|
||||||
# Authenticate the gcloud tool with your account.
|
|
||||||
$ gcloud auth login
|
|
||||||
|
|
||||||
# Create the indexes used in the datastore integration tests.
|
|
||||||
$ gcloud preview datastore create-indexes datastore/testdata/index.yaml
|
|
||||||
|
|
||||||
# Create a Google Cloud storage bucket with the same name as your test project,
|
|
||||||
# and with the Stackdriver Logging service account as owner, for the sink
|
|
||||||
# integration tests in logging.
|
|
||||||
$ gsutil mb gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
|
|
||||||
$ gsutil acl ch -g cloud-logs@google.com:O gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
|
|
||||||
|
|
||||||
# Create a PubSub topic for integration tests of storage notifications.
|
|
||||||
$ gcloud beta pubsub topics create go-storage-notification-test
|
|
||||||
|
|
||||||
# Create a Spanner instance for the spanner integration tests.
|
|
||||||
$ gcloud beta spanner instances create go-integration-test --config regional-us-central1 --nodes 1 --description 'Instance for go client test'
|
|
||||||
# NOTE: Spanner instances are priced by the node-hour, so you may want to delete
|
|
||||||
# the instance after testing with 'gcloud beta spanner instances delete'.
|
|
||||||
|
|
||||||
# For Storage integration tests:
|
|
||||||
# Enable KMS for your project in the Cloud Console.
|
|
||||||
# Create a KMS keyring, in the same location as the default location for your project's buckets.
|
|
||||||
$ gcloud kms keyrings create MY_KEYRING --location MY_LOCATION
|
|
||||||
# Create two keys in the keyring, named key1 and key2.
|
|
||||||
$ gcloud kms keys create key1 --keyring MY_KEYRING --location MY_LOCATION --purpose encryption
|
|
||||||
$ gcloud kms keys create key2 --keyring MY_KEYRING --location MY_LOCATION --purpose encryption
|
|
||||||
# As mentioned above, set the GCLOUD_TESTS_GOLANG_KEYRING environment variable.
|
|
||||||
$ export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/MY_LOCATION/keyRings/MY_KEYRING
|
|
||||||
# Authorize Google Cloud Storage to encrypt and decrypt using key1.
|
|
||||||
gsutil kms authorize -p $GCLOUD_TESTS_GOLANG_PROJECT_ID -k $GCLOUD_TESTS_GOLANG_KEYRING/cryptoKeys/key1
|
|
||||||
```
|
|
||||||
|
|
||||||
Once you've done the necessary setup, you can run the integration tests by running:
|
|
||||||
|
|
||||||
``` sh
|
|
||||||
$ go test -v cloud.google.com/go/...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Contributor License Agreements
|
|
||||||
|
|
||||||
Before we can accept your pull requests you'll need to sign a Contributor
|
|
||||||
License Agreement (CLA):
|
|
||||||
|
|
||||||
- **If you are an individual writing original source code** and **you own the
|
|
||||||
intellectual property**, then you'll need to sign an [individual CLA][indvcla].
|
|
||||||
- **If you work for a company that wants to allow you to contribute your
|
|
||||||
work**, then you'll need to sign a [corporate CLA][corpcla].
|
|
||||||
|
|
||||||
You can sign these electronically (just scroll to the bottom). After that,
|
|
||||||
we'll be able to accept your pull requests.
|
|
||||||
|
|
||||||
## Contributor Code of Conduct
|
|
||||||
|
|
||||||
As contributors and maintainers of this project,
|
|
||||||
and in the interest of fostering an open and welcoming community,
|
|
||||||
we pledge to respect all people who contribute through reporting issues,
|
|
||||||
posting feature requests, updating documentation,
|
|
||||||
submitting pull requests or patches, and other activities.
|
|
||||||
|
|
||||||
We are committed to making participation in this project
|
|
||||||
a harassment-free experience for everyone,
|
|
||||||
regardless of level of experience, gender, gender identity and expression,
|
|
||||||
sexual orientation, disability, personal appearance,
|
|
||||||
body size, race, ethnicity, age, religion, or nationality.
|
|
||||||
|
|
||||||
Examples of unacceptable behavior by participants include:
|
|
||||||
|
|
||||||
* The use of sexualized language or imagery
|
|
||||||
* Personal attacks
|
|
||||||
* Trolling or insulting/derogatory comments
|
|
||||||
* Public or private harassment
|
|
||||||
* Publishing other's private information,
|
|
||||||
such as physical or electronic
|
|
||||||
addresses, without explicit permission
|
|
||||||
* Other unethical or unprofessional conduct.
|
|
||||||
|
|
||||||
Project maintainers have the right and responsibility to remove, edit, or reject
|
|
||||||
comments, commits, code, wiki edits, issues, and other contributions
|
|
||||||
that are not aligned to this Code of Conduct.
|
|
||||||
By adopting this Code of Conduct,
|
|
||||||
project maintainers commit themselves to fairly and consistently
|
|
||||||
applying these principles to every aspect of managing this project.
|
|
||||||
Project maintainers who do not follow or enforce the Code of Conduct
|
|
||||||
may be permanently removed from the project team.
|
|
||||||
|
|
||||||
This code of conduct applies both within project spaces and in public spaces
|
|
||||||
when an individual is representing the project or its community.
|
|
||||||
|
|
||||||
Instances of abusive, harassing, or otherwise unacceptable behavior
|
|
||||||
may be reported by opening an issue
|
|
||||||
or contacting one or more of the project maintainers.
|
|
||||||
|
|
||||||
This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
|
|
||||||
available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
|
|
||||||
|
|
||||||
[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/
|
|
||||||
[indvcla]: https://developers.google.com/open-source/cla/individual
|
|
||||||
[corpcla]: https://developers.google.com/open-source/cla/corporate
|
|
||||||
40
vendor/cloud.google.com/go/CONTRIBUTORS
generated
vendored
40
vendor/cloud.google.com/go/CONTRIBUTORS
generated
vendored
@ -1,40 +0,0 @@
|
|||||||
# People who have agreed to one of the CLAs and can contribute patches.
|
|
||||||
# The AUTHORS file lists the copyright holders; this file
|
|
||||||
# lists people. For example, Google employees are listed here
|
|
||||||
# but not in AUTHORS, because Google holds the copyright.
|
|
||||||
#
|
|
||||||
# https://developers.google.com/open-source/cla/individual
|
|
||||||
# https://developers.google.com/open-source/cla/corporate
|
|
||||||
#
|
|
||||||
# Names should be added to this file as:
|
|
||||||
# Name <email address>
|
|
||||||
|
|
||||||
# Keep the list alphabetically sorted.
|
|
||||||
|
|
||||||
Alexis Hunt <lexer@google.com>
|
|
||||||
Andreas Litt <andreas.litt@gmail.com>
|
|
||||||
Andrew Gerrand <adg@golang.org>
|
|
||||||
Brad Fitzpatrick <bradfitz@golang.org>
|
|
||||||
Burcu Dogan <jbd@google.com>
|
|
||||||
Dave Day <djd@golang.org>
|
|
||||||
David Sansome <me@davidsansome.com>
|
|
||||||
David Symonds <dsymonds@golang.org>
|
|
||||||
Filippo Valsorda <hi@filippo.io>
|
|
||||||
Glenn Lewis <gmlewis@google.com>
|
|
||||||
Ingo Oeser <nightlyone@googlemail.com>
|
|
||||||
James Hall <james.hall@shopify.com>
|
|
||||||
Johan Euphrosine <proppy@google.com>
|
|
||||||
Jonathan Amsterdam <jba@google.com>
|
|
||||||
Kunpei Sakai <namusyaka@gmail.com>
|
|
||||||
Luna Duclos <luna.duclos@palmstonegames.com>
|
|
||||||
Magnus Hiie <magnus.hiie@gmail.com>
|
|
||||||
Mario Castro <mariocaster@gmail.com>
|
|
||||||
Michael McGreevy <mcgreevy@golang.org>
|
|
||||||
Omar Jarjur <ojarjur@google.com>
|
|
||||||
Paweł Knap <pawelknap88@gmail.com>
|
|
||||||
Péter Szilágyi <peterke@gmail.com>
|
|
||||||
Sarah Adams <shadams@google.com>
|
|
||||||
Thanatat Tamtan <acoshift@gmail.com>
|
|
||||||
Toby Burress <kurin@google.com>
|
|
||||||
Tuo Shan <shantuo@google.com>
|
|
||||||
Tyler Treat <ttreat31@gmail.com>
|
|
||||||
202
vendor/cloud.google.com/go/LICENSE
generated
vendored
202
vendor/cloud.google.com/go/LICENSE
generated
vendored
@ -1,202 +0,0 @@
|
|||||||
|
|
||||||
Apache License
|
|
||||||
Version 2.0, January 2004
|
|
||||||
http://www.apache.org/licenses/
|
|
||||||
|
|
||||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
||||||
|
|
||||||
1. Definitions.
|
|
||||||
|
|
||||||
"License" shall mean the terms and conditions for use, reproduction,
|
|
||||||
and distribution as defined by Sections 1 through 9 of this document.
|
|
||||||
|
|
||||||
"Licensor" shall mean the copyright owner or entity authorized by
|
|
||||||
the copyright owner that is granting the License.
|
|
||||||
|
|
||||||
"Legal Entity" shall mean the union of the acting entity and all
|
|
||||||
other entities that control, are controlled by, or are under common
|
|
||||||
control with that entity. For the purposes of this definition,
|
|
||||||
"control" means (i) the power, direct or indirect, to cause the
|
|
||||||
direction or management of such entity, whether by contract or
|
|
||||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
||||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
||||||
|
|
||||||
"You" (or "Your") shall mean an individual or Legal Entity
|
|
||||||
exercising permissions granted by this License.
|
|
||||||
|
|
||||||
"Source" form shall mean the preferred form for making modifications,
|
|
||||||
including but not limited to software source code, documentation
|
|
||||||
source, and configuration files.
|
|
||||||
|
|
||||||
"Object" form shall mean any form resulting from mechanical
|
|
||||||
transformation or translation of a Source form, including but
|
|
||||||
not limited to compiled object code, generated documentation,
|
|
||||||
and conversions to other media types.
|
|
||||||
|
|
||||||
"Work" shall mean the work of authorship, whether in Source or
|
|
||||||
Object form, made available under the License, as indicated by a
|
|
||||||
copyright notice that is included in or attached to the work
|
|
||||||
(an example is provided in the Appendix below).
|
|
||||||
|
|
||||||
"Derivative Works" shall mean any work, whether in Source or Object
|
|
||||||
form, that is based on (or derived from) the Work and for which the
|
|
||||||
editorial revisions, annotations, elaborations, or other modifications
|
|
||||||
represent, as a whole, an original work of authorship. For the purposes
|
|
||||||
of this License, Derivative Works shall not include works that remain
|
|
||||||
separable from, or merely link (or bind by name) to the interfaces of,
|
|
||||||
the Work and Derivative Works thereof.
|
|
||||||
|
|
||||||
"Contribution" shall mean any work of authorship, including
|
|
||||||
the original version of the Work and any modifications or additions
|
|
||||||
to that Work or Derivative Works thereof, that is intentionally
|
|
||||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
||||||
or by an individual or Legal Entity authorized to submit on behalf of
|
|
||||||
the copyright owner. For the purposes of this definition, "submitted"
|
|
||||||
means any form of electronic, verbal, or written communication sent
|
|
||||||
to the Licensor or its representatives, including but not limited to
|
|
||||||
communication on electronic mailing lists, source code control systems,
|
|
||||||
and issue tracking systems that are managed by, or on behalf of, the
|
|
||||||
Licensor for the purpose of discussing and improving the Work, but
|
|
||||||
excluding communication that is conspicuously marked or otherwise
|
|
||||||
designated in writing by the copyright owner as "Not a Contribution."
|
|
||||||
|
|
||||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
||||||
on behalf of whom a Contribution has been received by Licensor and
|
|
||||||
subsequently incorporated within the Work.
|
|
||||||
|
|
||||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
copyright license to reproduce, prepare Derivative Works of,
|
|
||||||
publicly display, publicly perform, sublicense, and distribute the
|
|
||||||
Work and such Derivative Works in Source or Object form.
|
|
||||||
|
|
||||||
3. Grant of Patent License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
(except as stated in this section) patent license to make, have made,
|
|
||||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
||||||
where such license applies only to those patent claims licensable
|
|
||||||
by such Contributor that are necessarily infringed by their
|
|
||||||
Contribution(s) alone or by combination of their Contribution(s)
|
|
||||||
with the Work to which such Contribution(s) was submitted. If You
|
|
||||||
institute patent litigation against any entity (including a
|
|
||||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
||||||
or a Contribution incorporated within the Work constitutes direct
|
|
||||||
or contributory patent infringement, then any patent licenses
|
|
||||||
granted to You under this License for that Work shall terminate
|
|
||||||
as of the date such litigation is filed.
|
|
||||||
|
|
||||||
4. Redistribution. You may reproduce and distribute copies of the
|
|
||||||
Work or Derivative Works thereof in any medium, with or without
|
|
||||||
modifications, and in Source or Object form, provided that You
|
|
||||||
meet the following conditions:
|
|
||||||
|
|
||||||
(a) You must give any other recipients of the Work or
|
|
||||||
Derivative Works a copy of this License; and
|
|
||||||
|
|
||||||
(b) You must cause any modified files to carry prominent notices
|
|
||||||
stating that You changed the files; and
|
|
||||||
|
|
||||||
(c) You must retain, in the Source form of any Derivative Works
|
|
||||||
that You distribute, all copyright, patent, trademark, and
|
|
||||||
attribution notices from the Source form of the Work,
|
|
||||||
excluding those notices that do not pertain to any part of
|
|
||||||
the Derivative Works; and
|
|
||||||
|
|
||||||
(d) If the Work includes a "NOTICE" text file as part of its
|
|
||||||
distribution, then any Derivative Works that You distribute must
|
|
||||||
include a readable copy of the attribution notices contained
|
|
||||||
within such NOTICE file, excluding those notices that do not
|
|
||||||
pertain to any part of the Derivative Works, in at least one
|
|
||||||
of the following places: within a NOTICE text file distributed
|
|
||||||
as part of the Derivative Works; within the Source form or
|
|
||||||
documentation, if provided along with the Derivative Works; or,
|
|
||||||
within a display generated by the Derivative Works, if and
|
|
||||||
wherever such third-party notices normally appear. The contents
|
|
||||||
of the NOTICE file are for informational purposes only and
|
|
||||||
do not modify the License. You may add Your own attribution
|
|
||||||
notices within Derivative Works that You distribute, alongside
|
|
||||||
or as an addendum to the NOTICE text from the Work, provided
|
|
||||||
that such additional attribution notices cannot be construed
|
|
||||||
as modifying the License.
|
|
||||||
|
|
||||||
You may add Your own copyright statement to Your modifications and
|
|
||||||
may provide additional or different license terms and conditions
|
|
||||||
for use, reproduction, or distribution of Your modifications, or
|
|
||||||
for any such Derivative Works as a whole, provided Your use,
|
|
||||||
reproduction, and distribution of the Work otherwise complies with
|
|
||||||
the conditions stated in this License.
|
|
||||||
|
|
||||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
||||||
any Contribution intentionally submitted for inclusion in the Work
|
|
||||||
by You to the Licensor shall be under the terms and conditions of
|
|
||||||
this License, without any additional terms or conditions.
|
|
||||||
Notwithstanding the above, nothing herein shall supersede or modify
|
|
||||||
the terms of any separate license agreement you may have executed
|
|
||||||
with Licensor regarding such Contributions.
|
|
||||||
|
|
||||||
6. Trademarks. This License does not grant permission to use the trade
|
|
||||||
names, trademarks, service marks, or product names of the Licensor,
|
|
||||||
except as required for reasonable and customary use in describing the
|
|
||||||
origin of the Work and reproducing the content of the NOTICE file.
|
|
||||||
|
|
||||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
||||||
agreed to in writing, Licensor provides the Work (and each
|
|
||||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
implied, including, without limitation, any warranties or conditions
|
|
||||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
||||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
||||||
appropriateness of using or redistributing the Work and assume any
|
|
||||||
risks associated with Your exercise of permissions under this License.
|
|
||||||
|
|
||||||
8. Limitation of Liability. In no event and under no legal theory,
|
|
||||||
whether in tort (including negligence), contract, or otherwise,
|
|
||||||
unless required by applicable law (such as deliberate and grossly
|
|
||||||
negligent acts) or agreed to in writing, shall any Contributor be
|
|
||||||
liable to You for damages, including any direct, indirect, special,
|
|
||||||
incidental, or consequential damages of any character arising as a
|
|
||||||
result of this License or out of the use or inability to use the
|
|
||||||
Work (including but not limited to damages for loss of goodwill,
|
|
||||||
work stoppage, computer failure or malfunction, or any and all
|
|
||||||
other commercial damages or losses), even if such Contributor
|
|
||||||
has been advised of the possibility of such damages.
|
|
||||||
|
|
||||||
9. Accepting Warranty or Additional Liability. While redistributing
|
|
||||||
the Work or Derivative Works thereof, You may choose to offer,
|
|
||||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
||||||
or other liability obligations and/or rights consistent with this
|
|
||||||
License. However, in accepting such obligations, You may act only
|
|
||||||
on Your own behalf and on Your sole responsibility, not on behalf
|
|
||||||
of any other Contributor, and only if You agree to indemnify,
|
|
||||||
defend, and hold each Contributor harmless for any liability
|
|
||||||
incurred by, or claims asserted against, such Contributor by reason
|
|
||||||
of your accepting any such warranty or additional liability.
|
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
APPENDIX: How to apply the Apache License to your work.
|
|
||||||
|
|
||||||
To apply the Apache License to your work, attach the following
|
|
||||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
||||||
replaced with your own identifying information. (Don't include
|
|
||||||
the brackets!) The text should be enclosed in the appropriate
|
|
||||||
comment syntax for the file format. We also recommend that a
|
|
||||||
file or class name and description of purpose be included on the
|
|
||||||
same "printed page" as the copyright notice for easier
|
|
||||||
identification within third-party archives.
|
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
54
vendor/cloud.google.com/go/MIGRATION.md
generated
vendored
54
vendor/cloud.google.com/go/MIGRATION.md
generated
vendored
@ -1,54 +0,0 @@
|
|||||||
# Code Changes
|
|
||||||
|
|
||||||
## v0.10.0
|
|
||||||
|
|
||||||
- pubsub: Replace
|
|
||||||
|
|
||||||
```
|
|
||||||
sub.ModifyPushConfig(ctx, pubsub.PushConfig{Endpoint: "https://example.com/push"})
|
|
||||||
```
|
|
||||||
|
|
||||||
with
|
|
||||||
|
|
||||||
```
|
|
||||||
sub.Update(ctx, pubsub.SubscriptionConfigToUpdate{
|
|
||||||
PushConfig: &pubsub.PushConfig{Endpoint: "https://example.com/push"},
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
- trace: traceGRPCServerInterceptor will be provided from *trace.Client.
|
|
||||||
Given an initialized `*trace.Client` named `tc`, instead of
|
|
||||||
|
|
||||||
```
|
|
||||||
s := grpc.NewServer(grpc.UnaryInterceptor(trace.GRPCServerInterceptor(tc)))
|
|
||||||
```
|
|
||||||
|
|
||||||
write
|
|
||||||
|
|
||||||
```
|
|
||||||
s := grpc.NewServer(grpc.UnaryInterceptor(tc.GRPCServerInterceptor()))
|
|
||||||
```
|
|
||||||
|
|
||||||
- trace trace.GRPCClientInterceptor will also provided from *trace.Client.
|
|
||||||
Instead of
|
|
||||||
|
|
||||||
```
|
|
||||||
conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(trace.GRPCClientInterceptor()))
|
|
||||||
```
|
|
||||||
|
|
||||||
write
|
|
||||||
|
|
||||||
```
|
|
||||||
conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor()))
|
|
||||||
```
|
|
||||||
|
|
||||||
- trace: We removed the deprecated `trace.EnableGRPCTracing`. Use the gRPC
|
|
||||||
interceptor as a dial option as shown below when initializing Cloud package
|
|
||||||
clients:
|
|
||||||
|
|
||||||
```
|
|
||||||
c, err := pubsub.NewClient(ctx, "project-id", option.WithGRPCDialOption(grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor())))
|
|
||||||
if err != nil {
|
|
||||||
...
|
|
||||||
}
|
|
||||||
```
|
|
||||||
660
vendor/cloud.google.com/go/README.md
generated
vendored
660
vendor/cloud.google.com/go/README.md
generated
vendored
@ -1,660 +0,0 @@
|
|||||||
# Google Cloud Client Libraries for Go
|
|
||||||
|
|
||||||
[](https://godoc.org/cloud.google.com/go)
|
|
||||||
|
|
||||||
Go packages for [Google Cloud Platform](https://cloud.google.com) services.
|
|
||||||
|
|
||||||
``` go
|
|
||||||
import "cloud.google.com/go"
|
|
||||||
```
|
|
||||||
|
|
||||||
To install the packages on your system, *do not clone the repo*. Instead use
|
|
||||||
|
|
||||||
```
|
|
||||||
$ go get -u cloud.google.com/go/...
|
|
||||||
```
|
|
||||||
|
|
||||||
**NOTE:** Some of these packages are under development, and may occasionally
|
|
||||||
make backwards-incompatible changes.
|
|
||||||
|
|
||||||
**NOTE:** Github repo is a mirror of [https://code.googlesource.com/gocloud](https://code.googlesource.com/gocloud).
|
|
||||||
|
|
||||||
* [News](#news)
|
|
||||||
* [Supported APIs](#supported-apis)
|
|
||||||
* [Go Versions Supported](#go-versions-supported)
|
|
||||||
* [Authorization](#authorization)
|
|
||||||
* [Cloud Datastore](#cloud-datastore-)
|
|
||||||
* [Cloud Storage](#cloud-storage-)
|
|
||||||
* [Cloud Pub/Sub](#cloud-pub-sub-)
|
|
||||||
* [Cloud BigQuery](#cloud-bigquery-)
|
|
||||||
* [Stackdriver Logging](#stackdriver-logging-)
|
|
||||||
* [Cloud Spanner](#cloud-spanner-)
|
|
||||||
|
|
||||||
|
|
||||||
## News
|
|
||||||
|
|
||||||
_May 18, 2018_
|
|
||||||
|
|
||||||
*v0.23.0*
|
|
||||||
|
|
||||||
- bigquery: Add DDL stats to query statistics.
|
|
||||||
- bigtable:
|
|
||||||
- cbt: Add cells-per-column limit for row lookup.
|
|
||||||
- cbt: Make it possible to combine read filters.
|
|
||||||
- dlp: v2beta2 client removed. Use the v2 client instead.
|
|
||||||
- firestore, spanner: Fix compilation errors due to protobuf changes.
|
|
||||||
|
|
||||||
_May 8, 2018_
|
|
||||||
|
|
||||||
*v0.22.0*
|
|
||||||
|
|
||||||
- bigtable:
|
|
||||||
- cbt: Support cells per column limit for row read.
|
|
||||||
- bttest: Correctly handle empty RowSet.
|
|
||||||
- Fix ReadModifyWrite operation in emulator.
|
|
||||||
- Fix API path in GetCluster.
|
|
||||||
|
|
||||||
- bigquery:
|
|
||||||
- BEHAVIOR CHANGE: Retry on 503 status code.
|
|
||||||
- Add dataset.DeleteWithContents.
|
|
||||||
- Add SchemaUpdateOptions for query jobs.
|
|
||||||
- Add Timeline to QueryStatistics.
|
|
||||||
- Add more stats to ExplainQueryStage.
|
|
||||||
- Support Parquet data format.
|
|
||||||
|
|
||||||
- datastore:
|
|
||||||
- Support omitempty for times.
|
|
||||||
|
|
||||||
- dlp:
|
|
||||||
- **BREAKING CHANGE:** Remove v1beta1 client. Please migrate to the v2 client,
|
|
||||||
which is now out of beta.
|
|
||||||
- Add v2 client.
|
|
||||||
|
|
||||||
- firestore:
|
|
||||||
- BEHAVIOR CHANGE: Treat set({}, MergeAll) as valid.
|
|
||||||
|
|
||||||
- iam:
|
|
||||||
- Support JWT signing via SignJwt callopt.
|
|
||||||
|
|
||||||
- profiler:
|
|
||||||
- BEHAVIOR CHANGE: PollForSerialOutput returns an error when context.Done.
|
|
||||||
- BEHAVIOR CHANGE: Increase the initial backoff to 1 minute.
|
|
||||||
- Avoid returning empty serial port output.
|
|
||||||
|
|
||||||
- pubsub:
|
|
||||||
- BEHAVIOR CHANGE: Don't backoff during next retryable error once stream is healthy.
|
|
||||||
- BEHAVIOR CHANGE: Don't backoff on EOF.
|
|
||||||
- pstest: Support Acknowledge and ModifyAckDeadline RPCs.
|
|
||||||
|
|
||||||
- redis:
|
|
||||||
- Add v1 beta Redis client.
|
|
||||||
|
|
||||||
- spanner:
|
|
||||||
- Support SessionLabels.
|
|
||||||
|
|
||||||
- speech:
|
|
||||||
- Add api v1 beta1 client.
|
|
||||||
|
|
||||||
- storage:
|
|
||||||
- BEHAVIOR CHANGE: Retry reads when retryable error occurs.
|
|
||||||
- Fix delete of object in requester-pays bucket.
|
|
||||||
- Support KMS integration.
|
|
||||||
|
|
||||||
_April 9, 2018_
|
|
||||||
|
|
||||||
*v0.21.0*
|
|
||||||
|
|
||||||
- bigquery:
|
|
||||||
- Add OpenCensus tracing.
|
|
||||||
|
|
||||||
- firestore:
|
|
||||||
- **BREAKING CHANGE:** If a document does not exist, return a DocumentSnapshot
|
|
||||||
whose Exists method returns false. DocumentRef.Get and Transaction.Get
|
|
||||||
return the non-nil DocumentSnapshot in addition to a NotFound error.
|
|
||||||
**DocumentRef.GetAll and Transaction.GetAll return a non-nil
|
|
||||||
DocumentSnapshot instead of nil.**
|
|
||||||
- Add DocumentIterator.Stop. **Call Stop whenever you are done with a
|
|
||||||
DocumentIterator.**
|
|
||||||
- Added Query.Snapshots and DocumentRef.Snapshots, which provide realtime
|
|
||||||
notification of updates. See https://cloud.google.com/firestore/docs/query-data/listen.
|
|
||||||
- Canceling an RPC now always returns a grpc.Status with codes.Canceled.
|
|
||||||
|
|
||||||
- spanner:
|
|
||||||
- Add `CommitTimestamp`, which supports inserting the commit timestamp of a
|
|
||||||
transaction into a column.
|
|
||||||
|
|
||||||
_March 22, 2018_
|
|
||||||
|
|
||||||
*v0.20.0*
|
|
||||||
|
|
||||||
- bigquery: Support SchemaUpdateOptions for load jobs.
|
|
||||||
|
|
||||||
- bigtable:
|
|
||||||
- Add SampleRowKeys.
|
|
||||||
- cbt: Support union, intersection GCPolicy.
|
|
||||||
- Retry admin RPCS.
|
|
||||||
- Add trace spans to retries.
|
|
||||||
|
|
||||||
- datastore: Add OpenCensus tracing.
|
|
||||||
|
|
||||||
- firestore:
|
|
||||||
- Fix queries involving Null and NaN.
|
|
||||||
- Allow Timestamp protobuffers for time values.
|
|
||||||
|
|
||||||
- logging: Add a WriteTimeout option.
|
|
||||||
|
|
||||||
- spanner: Support Batch API.
|
|
||||||
|
|
||||||
- storage: Add OpenCensus tracing.
|
|
||||||
|
|
||||||
|
|
||||||
_February 26, 2018_
|
|
||||||
|
|
||||||
*v0.19.0*
|
|
||||||
|
|
||||||
- bigquery:
|
|
||||||
- Support customer-managed encryption keys.
|
|
||||||
|
|
||||||
- bigtable:
|
|
||||||
- Improved emulator support.
|
|
||||||
- Support GetCluster.
|
|
||||||
|
|
||||||
- datastore:
|
|
||||||
- Add general mutations.
|
|
||||||
- Support pointer struct fields.
|
|
||||||
- Support transaction options.
|
|
||||||
|
|
||||||
- firestore:
|
|
||||||
- Add Transaction.GetAll.
|
|
||||||
- Support document cursors.
|
|
||||||
|
|
||||||
- logging:
|
|
||||||
- Support concurrent RPCs to the service.
|
|
||||||
- Support per-entry resources.
|
|
||||||
|
|
||||||
- profiler:
|
|
||||||
- Add config options to disable heap and thread profiling.
|
|
||||||
- Read the project ID from $GOOGLE_CLOUD_PROJECT when it's set.
|
|
||||||
|
|
||||||
- pubsub:
|
|
||||||
- BEHAVIOR CHANGE: Release flow control after ack/nack (instead of after the
|
|
||||||
callback returns).
|
|
||||||
- Add SubscriptionInProject.
|
|
||||||
- Add OpenCensus instrumentation for streaming pull.
|
|
||||||
|
|
||||||
- storage:
|
|
||||||
- Support CORS.
|
|
||||||
|
|
||||||
|
|
||||||
_January 18, 2018_
|
|
||||||
|
|
||||||
*v0.18.0*
|
|
||||||
|
|
||||||
- bigquery:
|
|
||||||
- Marked stable.
|
|
||||||
- Schema inference of nullable fields supported.
|
|
||||||
- Added TimePartitioning to QueryConfig.
|
|
||||||
|
|
||||||
- firestore: Data provided to DocumentRef.Set with a Merge option can contain
|
|
||||||
Delete sentinels.
|
|
||||||
|
|
||||||
- logging: Clients can accept parent resources other than projects.
|
|
||||||
|
|
||||||
- pubsub:
|
|
||||||
- pubsub/pstest: A lighweight fake for pubsub. Experimental; feedback welcome.
|
|
||||||
- Support updating more subscription metadata: AckDeadline,
|
|
||||||
RetainAckedMessages and RetentionDuration.
|
|
||||||
|
|
||||||
- oslogin/apiv1beta: New client for the Cloud OS Login API.
|
|
||||||
|
|
||||||
- rpcreplay: A package for recording and replaying gRPC traffic.
|
|
||||||
|
|
||||||
- spanner:
|
|
||||||
- Add a ReadWithOptions that supports a row limit, as well as an index.
|
|
||||||
- Support query plan and execution statistics.
|
|
||||||
- Added [OpenCensus](http://opencensus.io) support.
|
|
||||||
|
|
||||||
- storage: Clarify checksum validation for gzipped files (it is not validated
|
|
||||||
when the file is served uncompressed).
|
|
||||||
|
|
||||||
|
|
||||||
_December 11, 2017_
|
|
||||||
|
|
||||||
*v0.17.0*
|
|
||||||
|
|
||||||
- firestore BREAKING CHANGES:
|
|
||||||
- Remove UpdateMap and UpdateStruct; rename UpdatePaths to Update.
|
|
||||||
Change
|
|
||||||
`docref.UpdateMap(ctx, map[string]interface{}{"a.b", 1})`
|
|
||||||
to
|
|
||||||
`docref.Update(ctx, []firestore.Update{{Path: "a.b", Value: 1}})`
|
|
||||||
|
|
||||||
Change
|
|
||||||
`docref.UpdateStruct(ctx, []string{"Field"}, aStruct)`
|
|
||||||
to
|
|
||||||
`docref.Update(ctx, []firestore.Update{{Path: "Field", Value: aStruct.Field}})`
|
|
||||||
- Rename MergePaths to Merge; require args to be FieldPaths
|
|
||||||
- A value stored as an integer can be read into a floating-point field, and vice versa.
|
|
||||||
- bigtable/cmd/cbt:
|
|
||||||
- Support deleting a column.
|
|
||||||
- Add regex option for row read.
|
|
||||||
- spanner: Mark stable.
|
|
||||||
- storage:
|
|
||||||
- Add Reader.ContentEncoding method.
|
|
||||||
- Fix handling of SignedURL headers.
|
|
||||||
- bigquery:
|
|
||||||
- If Uploader.Put is called with no rows, it returns nil without making a
|
|
||||||
call.
|
|
||||||
- Schema inference supports the "nullable" option in struct tags for
|
|
||||||
non-required fields.
|
|
||||||
- TimePartitioning supports "Field".
|
|
||||||
|
|
||||||
|
|
||||||
[Older news](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/old-news.md)
|
|
||||||
|
|
||||||
## Supported APIs
|
|
||||||
|
|
||||||
Google API | Status | Package
|
|
||||||
---------------------------------|--------------|-----------------------------------------------------------
|
|
||||||
[BigQuery][cloud-bigquery] | stable | [`cloud.google.com/go/bigquery`][cloud-bigquery-ref]
|
|
||||||
[Bigtable][cloud-bigtable] | stable | [`cloud.google.com/go/bigtable`][cloud-bigtable-ref]
|
|
||||||
[Container][cloud-container] | alpha | [`cloud.google.com/go/container/apiv1`][cloud-container-ref]
|
|
||||||
[Data Loss Prevention][cloud-dlp]| alpha | [`cloud.google.com/go/dlp/apiv2beta1`][cloud-dlp-ref]
|
|
||||||
[Datastore][cloud-datastore] | stable | [`cloud.google.com/go/datastore`][cloud-datastore-ref]
|
|
||||||
[Debugger][cloud-debugger] | alpha | [`cloud.google.com/go/debugger/apiv2`][cloud-debugger-ref]
|
|
||||||
[ErrorReporting][cloud-errors] | alpha | [`cloud.google.com/go/errorreporting`][cloud-errors-ref]
|
|
||||||
[Firestore][cloud-firestore] | beta | [`cloud.google.com/go/firestore`][cloud-firestore-ref]
|
|
||||||
[Language][cloud-language] | stable | [`cloud.google.com/go/language/apiv1`][cloud-language-ref]
|
|
||||||
[Logging][cloud-logging] | stable | [`cloud.google.com/go/logging`][cloud-logging-ref]
|
|
||||||
[Monitoring][cloud-monitoring] | beta | [`cloud.google.com/go/monitoring/apiv3`][cloud-monitoring-ref]
|
|
||||||
[OS Login][cloud-oslogin] | alpha | [`cloud.google.com/compute/docs/oslogin/rest`][cloud-oslogin-ref]
|
|
||||||
[Pub/Sub][cloud-pubsub] | stable | [`cloud.google.com/go/pubsub`][cloud-pubsub-ref]
|
|
||||||
[Spanner][cloud-spanner] | stable | [`cloud.google.com/go/spanner`][cloud-spanner-ref]
|
|
||||||
[Speech][cloud-speech] | stable | [`cloud.google.com/go/speech/apiv1`][cloud-speech-ref]
|
|
||||||
[Storage][cloud-storage] | stable | [`cloud.google.com/go/storage`][cloud-storage-ref]
|
|
||||||
[Translation][cloud-translation] | stable | [`cloud.google.com/go/translate`][cloud-translation-ref]
|
|
||||||
[Video Intelligence][cloud-video]| beta | [`cloud.google.com/go/videointelligence/apiv1beta1`][cloud-video-ref]
|
|
||||||
[Vision][cloud-vision] | stable | [`cloud.google.com/go/vision/apiv1`][cloud-vision-ref]
|
|
||||||
|
|
||||||
|
|
||||||
> **Alpha status**: the API is still being actively developed. As a
|
|
||||||
> result, it might change in backward-incompatible ways and is not recommended
|
|
||||||
> for production use.
|
|
||||||
>
|
|
||||||
> **Beta status**: the API is largely complete, but still has outstanding
|
|
||||||
> features and bugs to be addressed. There may be minor backwards-incompatible
|
|
||||||
> changes where necessary.
|
|
||||||
>
|
|
||||||
> **Stable status**: the API is mature and ready for production use. We will
|
|
||||||
> continue addressing bugs and feature requests.
|
|
||||||
|
|
||||||
Documentation and examples are available at
|
|
||||||
https://godoc.org/cloud.google.com/go
|
|
||||||
|
|
||||||
Visit or join the
|
|
||||||
[google-api-go-announce group](https://groups.google.com/forum/#!forum/google-api-go-announce)
|
|
||||||
for updates on these packages.
|
|
||||||
|
|
||||||
## Go Versions Supported
|
|
||||||
|
|
||||||
We support the two most recent major versions of Go. If Google App Engine uses
|
|
||||||
an older version, we support that as well. You can see which versions are
|
|
||||||
currently supported by looking at the lines following `go:` in
|
|
||||||
[`.travis.yml`](.travis.yml).
|
|
||||||
|
|
||||||
## Authorization
|
|
||||||
|
|
||||||
By default, each API will use [Google Application Default Credentials][default-creds]
|
|
||||||
for authorization credentials used in calling the API endpoints. This will allow your
|
|
||||||
application to run in many environments without requiring explicit configuration.
|
|
||||||
|
|
||||||
[snip]:# (auth)
|
|
||||||
```go
|
|
||||||
client, err := storage.NewClient(ctx)
|
|
||||||
```
|
|
||||||
|
|
||||||
To authorize using a
|
|
||||||
[JSON key file](https://cloud.google.com/iam/docs/managing-service-account-keys),
|
|
||||||
pass
|
|
||||||
[`option.WithServiceAccountFile`](https://godoc.org/google.golang.org/api/option#WithServiceAccountFile)
|
|
||||||
to the `NewClient` function of the desired package. For example:
|
|
||||||
|
|
||||||
[snip]:# (auth-JSON)
|
|
||||||
```go
|
|
||||||
client, err := storage.NewClient(ctx, option.WithServiceAccountFile("path/to/keyfile.json"))
|
|
||||||
```
|
|
||||||
|
|
||||||
You can exert more control over authorization by using the
|
|
||||||
[`golang.org/x/oauth2`](https://godoc.org/golang.org/x/oauth2) package to
|
|
||||||
create an `oauth2.TokenSource`. Then pass
|
|
||||||
[`option.WithTokenSource`](https://godoc.org/google.golang.org/api/option#WithTokenSource)
|
|
||||||
to the `NewClient` function:
|
|
||||||
[snip]:# (auth-ts)
|
|
||||||
```go
|
|
||||||
tokenSource := ...
|
|
||||||
client, err := storage.NewClient(ctx, option.WithTokenSource(tokenSource))
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud Datastore [](https://godoc.org/cloud.google.com/go/datastore)
|
|
||||||
|
|
||||||
- [About Cloud Datastore][cloud-datastore]
|
|
||||||
- [Activating the API for your project][cloud-datastore-activation]
|
|
||||||
- [API documentation][cloud-datastore-docs]
|
|
||||||
- [Go client documentation](https://godoc.org/cloud.google.com/go/datastore)
|
|
||||||
- [Complete sample program](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/datastore/tasks)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `datastore.Client` to use throughout your application:
|
|
||||||
|
|
||||||
[snip]:# (datastore-1)
|
|
||||||
```go
|
|
||||||
client, err := datastore.NewClient(ctx, "my-project-id")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Then use that client to interact with the API:
|
|
||||||
|
|
||||||
[snip]:# (datastore-2)
|
|
||||||
```go
|
|
||||||
type Post struct {
|
|
||||||
Title string
|
|
||||||
Body string `datastore:",noindex"`
|
|
||||||
PublishedAt time.Time
|
|
||||||
}
|
|
||||||
keys := []*datastore.Key{
|
|
||||||
datastore.NameKey("Post", "post1", nil),
|
|
||||||
datastore.NameKey("Post", "post2", nil),
|
|
||||||
}
|
|
||||||
posts := []*Post{
|
|
||||||
{Title: "Post 1", Body: "...", PublishedAt: time.Now()},
|
|
||||||
{Title: "Post 2", Body: "...", PublishedAt: time.Now()},
|
|
||||||
}
|
|
||||||
if _, err := client.PutMulti(ctx, keys, posts); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud Storage [](https://godoc.org/cloud.google.com/go/storage)
|
|
||||||
|
|
||||||
- [About Cloud Storage][cloud-storage]
|
|
||||||
- [API documentation][cloud-storage-docs]
|
|
||||||
- [Go client documentation](https://godoc.org/cloud.google.com/go/storage)
|
|
||||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `storage.Client` to use throughout your application:
|
|
||||||
|
|
||||||
[snip]:# (storage-1)
|
|
||||||
```go
|
|
||||||
client, err := storage.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
[snip]:# (storage-2)
|
|
||||||
```go
|
|
||||||
// Read the object1 from bucket.
|
|
||||||
rc, err := client.Bucket("bucket").Object("object1").NewReader(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
defer rc.Close()
|
|
||||||
body, err := ioutil.ReadAll(rc)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud Pub/Sub [](https://godoc.org/cloud.google.com/go/pubsub)
|
|
||||||
|
|
||||||
- [About Cloud Pubsub][cloud-pubsub]
|
|
||||||
- [API documentation][cloud-pubsub-docs]
|
|
||||||
- [Go client documentation](https://godoc.org/cloud.google.com/go/pubsub)
|
|
||||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/pubsub)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `pubsub.Client` to use throughout your application:
|
|
||||||
|
|
||||||
[snip]:# (pubsub-1)
|
|
||||||
```go
|
|
||||||
client, err := pubsub.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Then use the client to publish and subscribe:
|
|
||||||
|
|
||||||
[snip]:# (pubsub-2)
|
|
||||||
```go
|
|
||||||
// Publish "hello world" on topic1.
|
|
||||||
topic := client.Topic("topic1")
|
|
||||||
res := topic.Publish(ctx, &pubsub.Message{
|
|
||||||
Data: []byte("hello world"),
|
|
||||||
})
|
|
||||||
// The publish happens asynchronously.
|
|
||||||
// Later, you can get the result from res:
|
|
||||||
...
|
|
||||||
msgID, err := res.Get(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use a callback to receive messages via subscription1.
|
|
||||||
sub := client.Subscription("subscription1")
|
|
||||||
err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
|
|
||||||
fmt.Println(m.Data)
|
|
||||||
m.Ack() // Acknowledge that we've consumed the message.
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud BigQuery [](https://godoc.org/cloud.google.com/go/bigquery)
|
|
||||||
|
|
||||||
- [About Cloud BigQuery][cloud-bigquery]
|
|
||||||
- [API documentation][cloud-bigquery-docs]
|
|
||||||
- [Go client documentation][cloud-bigquery-ref]
|
|
||||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/bigquery)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `bigquery.Client` to use throughout your application:
|
|
||||||
[snip]:# (bq-1)
|
|
||||||
```go
|
|
||||||
c, err := bigquery.NewClient(ctx, "my-project-ID")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Then use that client to interact with the API:
|
|
||||||
[snip]:# (bq-2)
|
|
||||||
```go
|
|
||||||
// Construct a query.
|
|
||||||
q := c.Query(`
|
|
||||||
SELECT year, SUM(number)
|
|
||||||
FROM [bigquery-public-data:usa_names.usa_1910_2013]
|
|
||||||
WHERE name = "William"
|
|
||||||
GROUP BY year
|
|
||||||
ORDER BY year
|
|
||||||
`)
|
|
||||||
// Execute the query.
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// Iterate through the results.
|
|
||||||
for {
|
|
||||||
var values []bigquery.Value
|
|
||||||
err := it.Next(&values)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(values)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
## Stackdriver Logging [](https://godoc.org/cloud.google.com/go/logging)
|
|
||||||
|
|
||||||
- [About Stackdriver Logging][cloud-logging]
|
|
||||||
- [API documentation][cloud-logging-docs]
|
|
||||||
- [Go client documentation][cloud-logging-ref]
|
|
||||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/logging)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `logging.Client` to use throughout your application:
|
|
||||||
[snip]:# (logging-1)
|
|
||||||
```go
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := logging.NewClient(ctx, "my-project")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Usually, you'll want to add log entries to a buffer to be periodically flushed
|
|
||||||
(automatically and asynchronously) to the Stackdriver Logging service.
|
|
||||||
[snip]:# (logging-2)
|
|
||||||
```go
|
|
||||||
logger := client.Logger("my-log")
|
|
||||||
logger.Log(logging.Entry{Payload: "something happened!"})
|
|
||||||
```
|
|
||||||
|
|
||||||
Close your client before your program exits, to flush any buffered log entries.
|
|
||||||
[snip]:# (logging-3)
|
|
||||||
```go
|
|
||||||
err = client.Close()
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cloud Spanner [](https://godoc.org/cloud.google.com/go/spanner)
|
|
||||||
|
|
||||||
- [About Cloud Spanner][cloud-spanner]
|
|
||||||
- [API documentation][cloud-spanner-docs]
|
|
||||||
- [Go client documentation](https://godoc.org/cloud.google.com/go/spanner)
|
|
||||||
|
|
||||||
### Example Usage
|
|
||||||
|
|
||||||
First create a `spanner.Client` to use throughout your application:
|
|
||||||
|
|
||||||
[snip]:# (spanner-1)
|
|
||||||
```go
|
|
||||||
client, err := spanner.NewClient(ctx, "projects/P/instances/I/databases/D")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
[snip]:# (spanner-2)
|
|
||||||
```go
|
|
||||||
// Simple Reads And Writes
|
|
||||||
_, err = client.Apply(ctx, []*spanner.Mutation{
|
|
||||||
spanner.Insert("Users",
|
|
||||||
[]string{"name", "email"},
|
|
||||||
[]interface{}{"alice", "a@example.com"})})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
row, err := client.Single().ReadRow(ctx, "Users",
|
|
||||||
spanner.Key{"alice"}, []string{"email"})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
## Contributing
|
|
||||||
|
|
||||||
Contributions are welcome. Please, see the
|
|
||||||
[CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md)
|
|
||||||
document for details. We're using Gerrit for our code reviews. Please don't open pull
|
|
||||||
requests against this repo, new pull requests will be automatically closed.
|
|
||||||
|
|
||||||
Please note that this project is released with a Contributor Code of Conduct.
|
|
||||||
By participating in this project you agree to abide by its terms.
|
|
||||||
See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md#contributor-code-of-conduct)
|
|
||||||
for more information.
|
|
||||||
|
|
||||||
[cloud-datastore]: https://cloud.google.com/datastore/
|
|
||||||
[cloud-datastore-ref]: https://godoc.org/cloud.google.com/go/datastore
|
|
||||||
[cloud-datastore-docs]: https://cloud.google.com/datastore/docs
|
|
||||||
[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
|
|
||||||
|
|
||||||
[cloud-firestore]: https://cloud.google.com/firestore/
|
|
||||||
[cloud-firestore-ref]: https://godoc.org/cloud.google.com/go/firestore
|
|
||||||
[cloud-firestore-docs]: https://cloud.google.com/firestore/docs
|
|
||||||
[cloud-firestore-activation]: https://cloud.google.com/firestore/docs/activate
|
|
||||||
|
|
||||||
[cloud-pubsub]: https://cloud.google.com/pubsub/
|
|
||||||
[cloud-pubsub-ref]: https://godoc.org/cloud.google.com/go/pubsub
|
|
||||||
[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs
|
|
||||||
|
|
||||||
[cloud-storage]: https://cloud.google.com/storage/
|
|
||||||
[cloud-storage-ref]: https://godoc.org/cloud.google.com/go/storage
|
|
||||||
[cloud-storage-docs]: https://cloud.google.com/storage/docs
|
|
||||||
[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
|
|
||||||
|
|
||||||
[cloud-bigtable]: https://cloud.google.com/bigtable/
|
|
||||||
[cloud-bigtable-ref]: https://godoc.org/cloud.google.com/go/bigtable
|
|
||||||
|
|
||||||
[cloud-bigquery]: https://cloud.google.com/bigquery/
|
|
||||||
[cloud-bigquery-docs]: https://cloud.google.com/bigquery/docs
|
|
||||||
[cloud-bigquery-ref]: https://godoc.org/cloud.google.com/go/bigquery
|
|
||||||
|
|
||||||
[cloud-logging]: https://cloud.google.com/logging/
|
|
||||||
[cloud-logging-docs]: https://cloud.google.com/logging/docs
|
|
||||||
[cloud-logging-ref]: https://godoc.org/cloud.google.com/go/logging
|
|
||||||
|
|
||||||
[cloud-monitoring]: https://cloud.google.com/monitoring/
|
|
||||||
[cloud-monitoring-ref]: https://godoc.org/cloud.google.com/go/monitoring/apiv3
|
|
||||||
|
|
||||||
[cloud-vision]: https://cloud.google.com/vision
|
|
||||||
[cloud-vision-ref]: https://godoc.org/cloud.google.com/go/vision/apiv1
|
|
||||||
|
|
||||||
[cloud-language]: https://cloud.google.com/natural-language
|
|
||||||
[cloud-language-ref]: https://godoc.org/cloud.google.com/go/language/apiv1
|
|
||||||
|
|
||||||
[cloud-oslogin]: https://cloud.google.com/compute/docs/oslogin/rest
|
|
||||||
[cloud-oslogin-ref]: https://cloud.google.com/compute/docs/oslogin/rest
|
|
||||||
|
|
||||||
[cloud-speech]: https://cloud.google.com/speech
|
|
||||||
[cloud-speech-ref]: https://godoc.org/cloud.google.com/go/speech/apiv1
|
|
||||||
|
|
||||||
[cloud-spanner]: https://cloud.google.com/spanner/
|
|
||||||
[cloud-spanner-ref]: https://godoc.org/cloud.google.com/go/spanner
|
|
||||||
[cloud-spanner-docs]: https://cloud.google.com/spanner/docs
|
|
||||||
|
|
||||||
[cloud-translation]: https://cloud.google.com/translation
|
|
||||||
[cloud-translation-ref]: https://godoc.org/cloud.google.com/go/translation
|
|
||||||
|
|
||||||
[cloud-video]: https://cloud.google.com/video-intelligence/
|
|
||||||
[cloud-video-ref]: https://godoc.org/cloud.google.com/go/videointelligence/apiv1beta1
|
|
||||||
|
|
||||||
[cloud-errors]: https://cloud.google.com/error-reporting/
|
|
||||||
[cloud-errors-ref]: https://godoc.org/cloud.google.com/go/errorreporting
|
|
||||||
|
|
||||||
[cloud-container]: https://cloud.google.com/containers/
|
|
||||||
[cloud-container-ref]: https://godoc.org/cloud.google.com/go/container/apiv1
|
|
||||||
|
|
||||||
[cloud-debugger]: https://cloud.google.com/debugger/
|
|
||||||
[cloud-debugger-ref]: https://godoc.org/cloud.google.com/go/debugger/apiv2
|
|
||||||
|
|
||||||
[cloud-dlp]: https://cloud.google.com/dlp/
|
|
||||||
[cloud-dlp-ref]: https://godoc.org/cloud.google.com/go/dlp/apiv2beta1
|
|
||||||
|
|
||||||
[default-creds]: https://developers.google.com/identity/protocols/application-default-credentials
|
|
||||||
13
vendor/cloud.google.com/go/RELEASING.md
generated
vendored
13
vendor/cloud.google.com/go/RELEASING.md
generated
vendored
@ -1,13 +0,0 @@
|
|||||||
# How to Release this Repo
|
|
||||||
|
|
||||||
1. Determine the current release version with `git tag -l`. It should look
|
|
||||||
something like `vX.Y.Z`. We'll call the current
|
|
||||||
version `$CV` and the new version `$NV`.
|
|
||||||
1. On master, run `git log $CV..` to list all the changes since the last
|
|
||||||
release.
|
|
||||||
1. Edit the News section of `README.md` to include a summary of the changes.
|
|
||||||
1. Mail the CL containing the `README.md` changes. When the CL is approved, submit it.
|
|
||||||
1. Without submitting any other CLs:
|
|
||||||
a. Switch to master.
|
|
||||||
b. Tag the repo with the next version: `git tag $NV`.
|
|
||||||
c. Push the tag: `git push origin $NV`.
|
|
||||||
32
vendor/cloud.google.com/go/appveyor.yml
generated
vendored
32
vendor/cloud.google.com/go/appveyor.yml
generated
vendored
@ -1,32 +0,0 @@
|
|||||||
# This file configures AppVeyor (http://www.appveyor.com),
|
|
||||||
# a Windows-based CI service similar to Travis.
|
|
||||||
|
|
||||||
# Identifier for this run
|
|
||||||
version: "{build}"
|
|
||||||
|
|
||||||
# Clone the repo into this path, which conforms to the standard
|
|
||||||
# Go workspace structure.
|
|
||||||
clone_folder: c:\gopath\src\cloud.google.com\go
|
|
||||||
|
|
||||||
environment:
|
|
||||||
GOPATH: c:\gopath
|
|
||||||
GCLOUD_TESTS_GOLANG_PROJECT_ID: dulcet-port-762
|
|
||||||
GCLOUD_TESTS_GOLANG_KEY: c:\gopath\src\cloud.google.com\go\key.json
|
|
||||||
KEYFILE_CONTENTS:
|
|
||||||
secure: IvRbDAhM2PIQqzVkjzJ4FjizUvoQ+c3vG/qhJQG+HlZ/L5KEkqLu+x6WjLrExrNMyGku4znB2jmbTrUW3Ob4sGG+R5vvqeQ3YMHCVIkw5CxY+/bUDkW5RZWsVbuCnNa/vKsWmCP+/sZW6ICe29yKJ2ZOb6QaauI4s9R6j+cqBbU9pumMGYFRb0Rw3uUU7DKmVFCy+NjTENZIlDP9rmjANgAzigowJJEb2Tg9sLlQKmQeKiBSRN8lKc5Nq60a+fIzHGKvql4eIitDDDpOpyHv15/Xr1BzFw2yDoiR4X1lng0u7q0X9RgX4VIYa6gT16NXBEmQgbuX8gh7SfPMp9RhiZD9sVUaV+yogEabYpyPnmUURo0hXwkctKaBkQlEmKvjHwF5dvbg8+yqGhwtjAgFNimXG3INrwQsfQsZskkQWanutbJf9xy50GyWWFZZdi0uT4oXP/b5P7aklPXKXsvrJKBh7RjEaqBrhi86IJwOjBspvoR4l2WmcQyxb2xzQS1pjbBJFQfYJJ8+JgsstTL8PBO9d4ybJC0li1Om1qnWxkaewvPxxuoHJ9LpRKof19yRYWBmhTXb2tTASKG/zslvl4fgG4DmQBS93WC7dsiGOhAraGw2eCTgd0lYZOhk1FjWl9TS80aktXxzH/7nTvem5ohm+eDl6O0wnTL4KXjQVNSQ1PyLn4lGRJ5MNGzBTRFWIr2API2rca4Fysyfh/UdmazPGlNbY9JPGqb9+F04QzLfqm+Zz/cHy59E7lOSMBlUI4KD6d6ZNNKNRH+/g9i+fSiyiXKugTfda8KBnWGyPwprxuWGYaiQUGUYOwJY5R6x5c4mjImAB310V+Wo33UbWFJiwxEDsiCNqW1meVkBzt2er26vh4qbgCUIQ3iM3gFPfHgy+QxkmIhic7Q1HYacQElt8AAP41M7cCKWCuZidegP37MBB//mjjiNt047ZSQEvB4tqsX/OvfbByVef+cbtVw9T0yjHvmCdPW1XrhyrCCgclu6oYYdbmc5D7BBDRbjjMWGv6YvceAbfGf6ukdB5PuV+TGEN/FoQ1QTRA6Aqf+3fLMg4mS4oyTfw5xyYNbv3qoyLPrp+BnxI53WB9p0hfMg4n9FD6NntBxjDq+Q3Lk/bjC/Y4MaRWdzbMzF9a0lgGfcw9DURlK5p7uGJC9vg34feNoQprxVEZRQ01cHLeob6eGkYm4HxSRx8JY39Mh+9wzJo+k/aIvFleNC3e35NOrkXr6wb5e42n2DwBdPqdNolTLtLFRglAL1LTpp27UjvjieWJAKfoDTR5CKl01sZqt0wPdLLcvsMj6CiPFmccUIOYeZMe86kLBD61Qa5F1EwkgO3Om2qSjW96FzL4skRc+BmU5RrHlAFSldR1wpUgtkUMv9vH5Cy+UJdcvpZ8KbmhZ2PsjF7ddJ1ve9RAw3cP325AyIMwZ77Ef1mgTM0NJze6eSW1qKlEsgt1FADPyeUu1NQTA2H2dueMPGlArWTSUgyWR9AdfpqouT7eg0JWI5w+yUZZC+/rPglYbt84oLmYpwuli0z8FyEQRPIc3EtkfWIv/yYgDr2TZ0N2KvGfpi/MAUWgxI1gleC2uKgEOEtuJthd3XZjF2NoE7IBqjQOINybcJOjyeB5vRLDY1FLuxYzdg1y1etkV4XQig/vje
|
|
||||||
|
|
||||||
install:
|
|
||||||
# Info for debugging.
|
|
||||||
- echo %PATH%
|
|
||||||
- go version
|
|
||||||
- go env
|
|
||||||
- go get -v -d -t ./...
|
|
||||||
|
|
||||||
|
|
||||||
# Provide a build script, or AppVeyor will call msbuild.
|
|
||||||
build_script:
|
|
||||||
- go install -v ./...
|
|
||||||
- echo %KEYFILE_CONTENTS% > %GCLOUD_TESTS_GOLANG_KEY%
|
|
||||||
|
|
||||||
test_script:
|
|
||||||
- go test -v ./...
|
|
||||||
72
vendor/cloud.google.com/go/authexample_test.go
generated
vendored
72
vendor/cloud.google.com/go/authexample_test.go
generated
vendored
@ -1,72 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cloud_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cloud.google.com/go/datastore"
|
|
||||||
"cloud.google.com/go/pubsub"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"golang.org/x/oauth2/google"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Google Application Default Credentials is the recommended way to authorize
|
|
||||||
// and authenticate clients.
|
|
||||||
//
|
|
||||||
// For information on how to create and obtain Application Default Credentials, see
|
|
||||||
// https://developers.google.com/identity/protocols/application-default-credentials.
|
|
||||||
func Example_applicationDefaultCredentials() {
|
|
||||||
client, err := datastore.NewClient(context.Background(), "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: handle error.
|
|
||||||
}
|
|
||||||
_ = client // Use the client.
|
|
||||||
}
|
|
||||||
|
|
||||||
// You can use a file with credentials to authenticate and authorize, such as a JSON
|
|
||||||
// key file associated with a Google service account. Service Account keys can be
|
|
||||||
// created and downloaded from
|
|
||||||
// https://console.developers.google.com/permissions/serviceaccounts.
|
|
||||||
//
|
|
||||||
// This example uses the Datastore client, but the same steps apply to
|
|
||||||
// the other client libraries underneath this package.
|
|
||||||
func Example_credentialsFile() {
|
|
||||||
client, err := datastore.NewClient(context.Background(),
|
|
||||||
"project-id", option.WithCredentialsFile("/path/to/service-account-key.json"))
|
|
||||||
if err != nil {
|
|
||||||
// TODO: handle error.
|
|
||||||
}
|
|
||||||
_ = client // Use the client.
|
|
||||||
}
|
|
||||||
|
|
||||||
// In some cases (for instance, you don't want to store secrets on disk), you can
|
|
||||||
// create credentials from in-memory JSON and use the WithCredentials option.
|
|
||||||
//
|
|
||||||
// The google package in this example is at golang.org/x/oauth2/google.
|
|
||||||
//
|
|
||||||
// This example uses the PubSub client, but the same steps apply to
|
|
||||||
// the other client libraries underneath this package.
|
|
||||||
func Example_credentialsFromJSON() {
|
|
||||||
ctx := context.Background()
|
|
||||||
creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), pubsub.ScopePubSub)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: handle error.
|
|
||||||
}
|
|
||||||
client, err := pubsub.NewClient(ctx, "project-id", option.WithCredentials(creds))
|
|
||||||
if err != nil {
|
|
||||||
// TODO: handle error.
|
|
||||||
}
|
|
||||||
_ = client // Use the client.
|
|
||||||
}
|
|
||||||
8
vendor/cloud.google.com/go/bigquery/benchmarks/README.md
generated
vendored
8
vendor/cloud.google.com/go/bigquery/benchmarks/README.md
generated
vendored
@ -1,8 +0,0 @@
|
|||||||
# BigQuery Benchmark
|
|
||||||
This directory contains benchmarks for BigQuery client.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
`go run bench.go -- <your project id> queries.json`
|
|
||||||
|
|
||||||
BigQuery service caches requests so the benchmark should be run
|
|
||||||
at least twice, disregarding the first result.
|
|
||||||
85
vendor/cloud.google.com/go/bigquery/benchmarks/bench.go
generated
vendored
85
vendor/cloud.google.com/go/bigquery/benchmarks/bench.go
generated
vendored
@ -1,85 +0,0 @@
|
|||||||
// Copyright 2017 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//+build ignore
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"flag"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigquery"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := bigquery.NewClient(ctx, flag.Arg(0))
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
queriesJSON, err := ioutil.ReadFile(flag.Arg(1))
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var queries []string
|
|
||||||
if err := json.Unmarshal(queriesJSON, &queries); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, q := range queries {
|
|
||||||
doQuery(ctx, c, q)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func doQuery(ctx context.Context, c *bigquery.Client, qt string) {
|
|
||||||
startTime := time.Now()
|
|
||||||
q := c.Query(qt)
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
numRows, numCols := 0, 0
|
|
||||||
var firstByte time.Duration
|
|
||||||
|
|
||||||
for {
|
|
||||||
var values []bigquery.Value
|
|
||||||
err := it.Next(&values)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
if numRows == 0 {
|
|
||||||
numCols = len(values)
|
|
||||||
firstByte = time.Since(startTime)
|
|
||||||
} else if numCols != len(values) {
|
|
||||||
log.Fatalf("got %d columns, want %d", len(values), numCols)
|
|
||||||
}
|
|
||||||
numRows++
|
|
||||||
}
|
|
||||||
log.Printf("query %q: %d rows, %d cols, first byte %f sec, total %f sec",
|
|
||||||
qt, numRows, numCols, firstByte.Seconds(), time.Since(startTime).Seconds())
|
|
||||||
}
|
|
||||||
10
vendor/cloud.google.com/go/bigquery/benchmarks/queries.json
generated
vendored
10
vendor/cloud.google.com/go/bigquery/benchmarks/queries.json
generated
vendored
@ -1,10 +0,0 @@
|
|||||||
[
|
|
||||||
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 10000",
|
|
||||||
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 100000",
|
|
||||||
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 1000000",
|
|
||||||
"SELECT title FROM `bigquery-public-data.samples.wikipedia` ORDER BY title LIMIT 1000",
|
|
||||||
"SELECT title, id, timestamp, contributor_ip FROM `bigquery-public-data.samples.wikipedia` WHERE title like 'Blo%' ORDER BY id",
|
|
||||||
"SELECT * FROM `bigquery-public-data.baseball.games_post_wide` ORDER BY gameId",
|
|
||||||
"SELECT * FROM `bigquery-public-data.samples.github_nested` WHERE repository.has_downloads ORDER BY repository.created_at LIMIT 10000",
|
|
||||||
"SELECT repo_name, path FROM `bigquery-public-data.github_repos.files` WHERE path LIKE '%.java' ORDER BY id LIMIT 1000000"
|
|
||||||
]
|
|
||||||
164
vendor/cloud.google.com/go/bigquery/bigquery.go
generated
vendored
164
vendor/cloud.google.com/go/bigquery/bigquery.go
generated
vendored
@ -1,164 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
gax "github.com/googleapis/gax-go"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal"
|
|
||||||
"cloud.google.com/go/internal/version"
|
|
||||||
|
|
||||||
"google.golang.org/api/googleapi"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
htransport "google.golang.org/api/transport/http"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
prodAddr = "https://www.googleapis.com/bigquery/v2/"
|
|
||||||
Scope = "https://www.googleapis.com/auth/bigquery"
|
|
||||||
userAgent = "gcloud-golang-bigquery/20160429"
|
|
||||||
)
|
|
||||||
|
|
||||||
var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
|
|
||||||
|
|
||||||
func setClientHeader(headers http.Header) {
|
|
||||||
headers.Set("x-goog-api-client", xGoogHeader)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client may be used to perform BigQuery operations.
|
|
||||||
type Client struct {
|
|
||||||
// Location, if set, will be used as the default location for all subsequent
|
|
||||||
// dataset creation and job operations. A location specified directly in one of
|
|
||||||
// those operations will override this value.
|
|
||||||
Location string
|
|
||||||
|
|
||||||
projectID string
|
|
||||||
bqs *bq.Service
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient constructs a new Client which can perform BigQuery operations.
|
|
||||||
// Operations performed via the client are billed to the specified GCP project.
|
|
||||||
func NewClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*Client, error) {
|
|
||||||
o := []option.ClientOption{
|
|
||||||
option.WithEndpoint(prodAddr),
|
|
||||||
option.WithScopes(Scope),
|
|
||||||
option.WithUserAgent(userAgent),
|
|
||||||
}
|
|
||||||
o = append(o, opts...)
|
|
||||||
httpClient, endpoint, err := htransport.NewClient(ctx, o...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("bigquery: dialing: %v", err)
|
|
||||||
}
|
|
||||||
bqs, err := bq.New(httpClient)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("bigquery: constructing client: %v", err)
|
|
||||||
}
|
|
||||||
bqs.BasePath = endpoint
|
|
||||||
c := &Client{
|
|
||||||
projectID: projectID,
|
|
||||||
bqs: bqs,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes any resources held by the client.
|
|
||||||
// Close should be called when the client is no longer needed.
|
|
||||||
// It need not be called at program exit.
|
|
||||||
func (c *Client) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calls the Jobs.Insert RPC and returns a Job.
|
|
||||||
func (c *Client) insertJob(ctx context.Context, job *bq.Job, media io.Reader) (*Job, error) {
|
|
||||||
call := c.bqs.Jobs.Insert(c.projectID, job).Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if media != nil {
|
|
||||||
call.Media(media)
|
|
||||||
}
|
|
||||||
var res *bq.Job
|
|
||||||
var err error
|
|
||||||
invoke := func() error {
|
|
||||||
res, err = call.Do()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// A job with a client-generated ID can be retried; the presence of the
|
|
||||||
// ID makes the insert operation idempotent.
|
|
||||||
// We don't retry if there is media, because it is an io.Reader. We'd
|
|
||||||
// have to read the contents and keep it in memory, and that could be expensive.
|
|
||||||
// TODO(jba): Look into retrying if media != nil.
|
|
||||||
if job.JobReference != nil && media == nil {
|
|
||||||
err = runWithRetry(ctx, invoke)
|
|
||||||
} else {
|
|
||||||
err = invoke()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToJob(res, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert a number of milliseconds since the Unix epoch to a time.Time.
|
|
||||||
// Treat an input of zero specially: convert it to the zero time,
|
|
||||||
// rather than the start of the epoch.
|
|
||||||
func unixMillisToTime(m int64) time.Time {
|
|
||||||
if m == 0 {
|
|
||||||
return time.Time{}
|
|
||||||
}
|
|
||||||
return time.Unix(0, m*1e6)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runWithRetry calls the function until it returns nil or a non-retryable error, or
|
|
||||||
// the context is done.
|
|
||||||
// See the similar function in ../storage/invoke.go. The main difference is the
|
|
||||||
// reason for retrying.
|
|
||||||
func runWithRetry(ctx context.Context, call func() error) error {
|
|
||||||
// These parameters match the suggestions in https://cloud.google.com/bigquery/sla.
|
|
||||||
backoff := gax.Backoff{
|
|
||||||
Initial: 1 * time.Second,
|
|
||||||
Max: 32 * time.Second,
|
|
||||||
Multiplier: 2,
|
|
||||||
}
|
|
||||||
return internal.Retry(ctx, backoff, func() (stop bool, err error) {
|
|
||||||
err = call()
|
|
||||||
if err == nil {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return !retryableError(err), err
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// This is the correct definition of retryable according to the BigQuery team. It
|
|
||||||
// also considers 502 ("Bad Gateway") and 503 ("Service Unavailable") errors
|
|
||||||
// retryable; these are returned by systems between the client and the BigQuery
|
|
||||||
// service.
|
|
||||||
func retryableError(err error) bool {
|
|
||||||
e, ok := err.(*googleapi.Error)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var reason string
|
|
||||||
if len(e.Errors) > 0 {
|
|
||||||
reason = e.Errors[0].Reason
|
|
||||||
}
|
|
||||||
return e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded"
|
|
||||||
}
|
|
||||||
106
vendor/cloud.google.com/go/bigquery/copy.go
generated
vendored
106
vendor/cloud.google.com/go/bigquery/copy.go
generated
vendored
@ -1,106 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CopyConfig holds the configuration for a copy job.
|
|
||||||
type CopyConfig struct {
|
|
||||||
// Srcs are the tables from which data will be copied.
|
|
||||||
Srcs []*Table
|
|
||||||
|
|
||||||
// Dst is the table into which the data will be copied.
|
|
||||||
Dst *Table
|
|
||||||
|
|
||||||
// CreateDisposition specifies the circumstances under which the destination table will be created.
|
|
||||||
// The default is CreateIfNeeded.
|
|
||||||
CreateDisposition TableCreateDisposition
|
|
||||||
|
|
||||||
// WriteDisposition specifies how existing data in the destination table is treated.
|
|
||||||
// The default is WriteEmpty.
|
|
||||||
WriteDisposition TableWriteDisposition
|
|
||||||
|
|
||||||
// The labels associated with this job.
|
|
||||||
Labels map[string]string
|
|
||||||
|
|
||||||
// Custom encryption configuration (e.g., Cloud KMS keys).
|
|
||||||
DestinationEncryptionConfig *EncryptionConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CopyConfig) toBQ() *bq.JobConfiguration {
|
|
||||||
var ts []*bq.TableReference
|
|
||||||
for _, t := range c.Srcs {
|
|
||||||
ts = append(ts, t.toBQ())
|
|
||||||
}
|
|
||||||
return &bq.JobConfiguration{
|
|
||||||
Labels: c.Labels,
|
|
||||||
Copy: &bq.JobConfigurationTableCopy{
|
|
||||||
CreateDisposition: string(c.CreateDisposition),
|
|
||||||
WriteDisposition: string(c.WriteDisposition),
|
|
||||||
DestinationTable: c.Dst.toBQ(),
|
|
||||||
DestinationEncryptionConfiguration: c.DestinationEncryptionConfig.toBQ(),
|
|
||||||
SourceTables: ts,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToCopyConfig(q *bq.JobConfiguration, c *Client) *CopyConfig {
|
|
||||||
cc := &CopyConfig{
|
|
||||||
Labels: q.Labels,
|
|
||||||
CreateDisposition: TableCreateDisposition(q.Copy.CreateDisposition),
|
|
||||||
WriteDisposition: TableWriteDisposition(q.Copy.WriteDisposition),
|
|
||||||
Dst: bqToTable(q.Copy.DestinationTable, c),
|
|
||||||
DestinationEncryptionConfig: bqToEncryptionConfig(q.Copy.DestinationEncryptionConfiguration),
|
|
||||||
}
|
|
||||||
for _, t := range q.Copy.SourceTables {
|
|
||||||
cc.Srcs = append(cc.Srcs, bqToTable(t, c))
|
|
||||||
}
|
|
||||||
return cc
|
|
||||||
}
|
|
||||||
|
|
||||||
// A Copier copies data into a BigQuery table from one or more BigQuery tables.
|
|
||||||
type Copier struct {
|
|
||||||
JobIDConfig
|
|
||||||
CopyConfig
|
|
||||||
c *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// CopierFrom returns a Copier which can be used to copy data into a
|
|
||||||
// BigQuery table from one or more BigQuery tables.
|
|
||||||
// The returned Copier may optionally be further configured before its Run method is called.
|
|
||||||
func (t *Table) CopierFrom(srcs ...*Table) *Copier {
|
|
||||||
return &Copier{
|
|
||||||
c: t.c,
|
|
||||||
CopyConfig: CopyConfig{
|
|
||||||
Srcs: srcs,
|
|
||||||
Dst: t,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run initiates a copy job.
|
|
||||||
func (c *Copier) Run(ctx context.Context) (*Job, error) {
|
|
||||||
return c.c.insertJob(ctx, c.newJob(), nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Copier) newJob() *bq.Job {
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: c.JobIDConfig.createJobRef(c.c),
|
|
||||||
Configuration: c.CopyConfig.toBQ(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
165
vendor/cloud.google.com/go/bigquery/copy_test.go
generated
vendored
165
vendor/cloud.google.com/go/bigquery/copy_test.go
generated
vendored
@ -1,165 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp/cmpopts"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func defaultCopyJob() *bq.Job {
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
|
|
||||||
Configuration: &bq.JobConfiguration{
|
|
||||||
Copy: &bq.JobConfigurationTableCopy{
|
|
||||||
DestinationTable: &bq.TableReference{
|
|
||||||
ProjectId: "d-project-id",
|
|
||||||
DatasetId: "d-dataset-id",
|
|
||||||
TableId: "d-table-id",
|
|
||||||
},
|
|
||||||
SourceTables: []*bq.TableReference{
|
|
||||||
{
|
|
||||||
ProjectId: "s-project-id",
|
|
||||||
DatasetId: "s-dataset-id",
|
|
||||||
TableId: "s-table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopy(t *testing.T) {
|
|
||||||
defer fixRandomID("RANDOM")()
|
|
||||||
testCases := []struct {
|
|
||||||
dst *Table
|
|
||||||
srcs []*Table
|
|
||||||
jobID string
|
|
||||||
location string
|
|
||||||
config CopyConfig
|
|
||||||
want *bq.Job
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
dst: &Table{
|
|
||||||
ProjectID: "d-project-id",
|
|
||||||
DatasetID: "d-dataset-id",
|
|
||||||
TableID: "d-table-id",
|
|
||||||
},
|
|
||||||
srcs: []*Table{
|
|
||||||
{
|
|
||||||
ProjectID: "s-project-id",
|
|
||||||
DatasetID: "s-dataset-id",
|
|
||||||
TableID: "s-table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: defaultCopyJob(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: &Table{
|
|
||||||
ProjectID: "d-project-id",
|
|
||||||
DatasetID: "d-dataset-id",
|
|
||||||
TableID: "d-table-id",
|
|
||||||
},
|
|
||||||
srcs: []*Table{
|
|
||||||
{
|
|
||||||
ProjectID: "s-project-id",
|
|
||||||
DatasetID: "s-dataset-id",
|
|
||||||
TableID: "s-table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
config: CopyConfig{
|
|
||||||
CreateDisposition: CreateNever,
|
|
||||||
WriteDisposition: WriteTruncate,
|
|
||||||
DestinationEncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultCopyJob()
|
|
||||||
j.Configuration.Labels = map[string]string{"a": "b"}
|
|
||||||
j.Configuration.Copy.CreateDisposition = "CREATE_NEVER"
|
|
||||||
j.Configuration.Copy.WriteDisposition = "WRITE_TRUNCATE"
|
|
||||||
j.Configuration.Copy.DestinationEncryptionConfiguration = &bq.EncryptionConfiguration{KmsKeyName: "keyName"}
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: &Table{
|
|
||||||
ProjectID: "d-project-id",
|
|
||||||
DatasetID: "d-dataset-id",
|
|
||||||
TableID: "d-table-id",
|
|
||||||
},
|
|
||||||
srcs: []*Table{
|
|
||||||
{
|
|
||||||
ProjectID: "s-project-id",
|
|
||||||
DatasetID: "s-dataset-id",
|
|
||||||
TableID: "s-table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
jobID: "job-id",
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultCopyJob()
|
|
||||||
j.JobReference.JobId = "job-id"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: &Table{
|
|
||||||
ProjectID: "d-project-id",
|
|
||||||
DatasetID: "d-dataset-id",
|
|
||||||
TableID: "d-table-id",
|
|
||||||
},
|
|
||||||
srcs: []*Table{
|
|
||||||
{
|
|
||||||
ProjectID: "s-project-id",
|
|
||||||
DatasetID: "s-dataset-id",
|
|
||||||
TableID: "s-table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
location: "asia-northeast1",
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultCopyJob()
|
|
||||||
j.JobReference.Location = "asia-northeast1"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
c := &Client{projectID: "client-project-id"}
|
|
||||||
for i, tc := range testCases {
|
|
||||||
tc.dst.c = c
|
|
||||||
copier := tc.dst.CopierFrom(tc.srcs...)
|
|
||||||
copier.JobID = tc.jobID
|
|
||||||
copier.Location = tc.location
|
|
||||||
tc.config.Srcs = tc.srcs
|
|
||||||
tc.config.Dst = tc.dst
|
|
||||||
copier.CopyConfig = tc.config
|
|
||||||
got := copier.newJob()
|
|
||||||
checkJob(t, i, got, tc.want)
|
|
||||||
|
|
||||||
jc, err := bqToJobConfig(got.Configuration, c)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("#%d: %v", i, err)
|
|
||||||
}
|
|
||||||
diff := testutil.Diff(jc.(*CopyConfig), &copier.CopyConfig,
|
|
||||||
cmpopts.IgnoreUnexported(Table{}))
|
|
||||||
if diff != "" {
|
|
||||||
t.Errorf("#%d: (got=-, want=+:\n%s", i, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
530
vendor/cloud.google.com/go/bigquery/dataset.go
generated
vendored
530
vendor/cloud.google.com/go/bigquery/dataset.go
generated
vendored
@ -1,530 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/optional"
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Dataset is a reference to a BigQuery dataset.
|
|
||||||
type Dataset struct {
|
|
||||||
ProjectID string
|
|
||||||
DatasetID string
|
|
||||||
c *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetMetadata contains information about a BigQuery dataset.
|
|
||||||
type DatasetMetadata struct {
|
|
||||||
// These fields can be set when creating a dataset.
|
|
||||||
Name string // The user-friendly name for this dataset.
|
|
||||||
Description string // The user-friendly description of this dataset.
|
|
||||||
Location string // The geo location of the dataset.
|
|
||||||
DefaultTableExpiration time.Duration // The default expiration time for new tables.
|
|
||||||
Labels map[string]string // User-provided labels.
|
|
||||||
Access []*AccessEntry // Access permissions.
|
|
||||||
|
|
||||||
// These fields are read-only.
|
|
||||||
CreationTime time.Time
|
|
||||||
LastModifiedTime time.Time // When the dataset or any of its tables were modified.
|
|
||||||
FullID string // The full dataset ID in the form projectID:datasetID.
|
|
||||||
|
|
||||||
// ETag is the ETag obtained when reading metadata. Pass it to Dataset.Update to
|
|
||||||
// ensure that the metadata hasn't changed since it was read.
|
|
||||||
ETag string
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetMetadataToUpdate is used when updating a dataset's metadata.
|
|
||||||
// Only non-nil fields will be updated.
|
|
||||||
type DatasetMetadataToUpdate struct {
|
|
||||||
Description optional.String // The user-friendly description of this table.
|
|
||||||
Name optional.String // The user-friendly name for this dataset.
|
|
||||||
|
|
||||||
// DefaultTableExpiration is the the default expiration time for new tables.
|
|
||||||
// If set to time.Duration(0), new tables never expire.
|
|
||||||
DefaultTableExpiration optional.Duration
|
|
||||||
|
|
||||||
// The entire access list. It is not possible to replace individual entries.
|
|
||||||
Access []*AccessEntry
|
|
||||||
|
|
||||||
labelUpdater
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dataset creates a handle to a BigQuery dataset in the client's project.
|
|
||||||
func (c *Client) Dataset(id string) *Dataset {
|
|
||||||
return c.DatasetInProject(c.projectID, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
|
|
||||||
func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
|
|
||||||
return &Dataset{
|
|
||||||
ProjectID: projectID,
|
|
||||||
DatasetID: datasetID,
|
|
||||||
c: c,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create creates a dataset in the BigQuery service. An error will be returned if the
|
|
||||||
// dataset already exists. Pass in a DatasetMetadata value to configure the dataset.
|
|
||||||
func (d *Dataset) Create(ctx context.Context, md *DatasetMetadata) (err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Dataset.Create")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
ds, err := md.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ds.DatasetReference = &bq.DatasetReference{DatasetId: d.DatasetID}
|
|
||||||
// Use Client.Location as a default.
|
|
||||||
if ds.Location == "" {
|
|
||||||
ds.Location = d.c.Location
|
|
||||||
}
|
|
||||||
call := d.c.bqs.Datasets.Insert(d.ProjectID, ds).Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
_, err = call.Do()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *DatasetMetadata) toBQ() (*bq.Dataset, error) {
|
|
||||||
ds := &bq.Dataset{}
|
|
||||||
if dm == nil {
|
|
||||||
return ds, nil
|
|
||||||
}
|
|
||||||
ds.FriendlyName = dm.Name
|
|
||||||
ds.Description = dm.Description
|
|
||||||
ds.Location = dm.Location
|
|
||||||
ds.DefaultTableExpirationMs = int64(dm.DefaultTableExpiration / time.Millisecond)
|
|
||||||
ds.Labels = dm.Labels
|
|
||||||
var err error
|
|
||||||
ds.Access, err = accessListToBQ(dm.Access)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !dm.CreationTime.IsZero() {
|
|
||||||
return nil, errors.New("bigquery: Dataset.CreationTime is not writable")
|
|
||||||
}
|
|
||||||
if !dm.LastModifiedTime.IsZero() {
|
|
||||||
return nil, errors.New("bigquery: Dataset.LastModifiedTime is not writable")
|
|
||||||
}
|
|
||||||
if dm.FullID != "" {
|
|
||||||
return nil, errors.New("bigquery: Dataset.FullID is not writable")
|
|
||||||
}
|
|
||||||
if dm.ETag != "" {
|
|
||||||
return nil, errors.New("bigquery: Dataset.ETag is not writable")
|
|
||||||
}
|
|
||||||
return ds, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func accessListToBQ(a []*AccessEntry) ([]*bq.DatasetAccess, error) {
|
|
||||||
var q []*bq.DatasetAccess
|
|
||||||
for _, e := range a {
|
|
||||||
a, err := e.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
q = append(q, a)
|
|
||||||
}
|
|
||||||
return q, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete deletes the dataset. Delete will fail if the dataset is not empty.
|
|
||||||
func (d *Dataset) Delete(ctx context.Context) (err error) {
|
|
||||||
return d.deleteInternal(ctx, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteWithContents deletes the dataset, as well as contained resources.
|
|
||||||
func (d *Dataset) DeleteWithContents(ctx context.Context) (err error) {
|
|
||||||
return d.deleteInternal(ctx, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Dataset) deleteInternal(ctx context.Context, deleteContents bool) (err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Dataset.Delete")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
call := d.c.bqs.Datasets.Delete(d.ProjectID, d.DatasetID).Context(ctx).DeleteContents(deleteContents)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
return call.Do()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Metadata fetches the metadata for the dataset.
|
|
||||||
func (d *Dataset) Metadata(ctx context.Context) (md *DatasetMetadata, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Dataset.Metadata")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
call := d.c.bqs.Datasets.Get(d.ProjectID, d.DatasetID).Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
var ds *bq.Dataset
|
|
||||||
if err := runWithRetry(ctx, func() (err error) {
|
|
||||||
ds, err = call.Do()
|
|
||||||
return err
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToDatasetMetadata(ds)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToDatasetMetadata(d *bq.Dataset) (*DatasetMetadata, error) {
|
|
||||||
dm := &DatasetMetadata{
|
|
||||||
CreationTime: unixMillisToTime(d.CreationTime),
|
|
||||||
LastModifiedTime: unixMillisToTime(d.LastModifiedTime),
|
|
||||||
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
|
|
||||||
Description: d.Description,
|
|
||||||
Name: d.FriendlyName,
|
|
||||||
FullID: d.Id,
|
|
||||||
Location: d.Location,
|
|
||||||
Labels: d.Labels,
|
|
||||||
ETag: d.Etag,
|
|
||||||
}
|
|
||||||
for _, a := range d.Access {
|
|
||||||
e, err := bqToAccessEntry(a, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
dm.Access = append(dm.Access, e)
|
|
||||||
}
|
|
||||||
return dm, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update modifies specific Dataset metadata fields.
|
|
||||||
// To perform a read-modify-write that protects against intervening reads,
|
|
||||||
// set the etag argument to the DatasetMetadata.ETag field from the read.
|
|
||||||
// Pass the empty string for etag for a "blind write" that will always succeed.
|
|
||||||
func (d *Dataset) Update(ctx context.Context, dm DatasetMetadataToUpdate, etag string) (md *DatasetMetadata, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Dataset.Update")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
ds, err := dm.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
call := d.c.bqs.Datasets.Patch(d.ProjectID, d.DatasetID, ds).Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if etag != "" {
|
|
||||||
call.Header().Set("If-Match", etag)
|
|
||||||
}
|
|
||||||
var ds2 *bq.Dataset
|
|
||||||
if err := runWithRetry(ctx, func() (err error) {
|
|
||||||
ds2, err = call.Do()
|
|
||||||
return err
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToDatasetMetadata(ds2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *DatasetMetadataToUpdate) toBQ() (*bq.Dataset, error) {
|
|
||||||
ds := &bq.Dataset{}
|
|
||||||
forceSend := func(field string) {
|
|
||||||
ds.ForceSendFields = append(ds.ForceSendFields, field)
|
|
||||||
}
|
|
||||||
|
|
||||||
if dm.Description != nil {
|
|
||||||
ds.Description = optional.ToString(dm.Description)
|
|
||||||
forceSend("Description")
|
|
||||||
}
|
|
||||||
if dm.Name != nil {
|
|
||||||
ds.FriendlyName = optional.ToString(dm.Name)
|
|
||||||
forceSend("FriendlyName")
|
|
||||||
}
|
|
||||||
if dm.DefaultTableExpiration != nil {
|
|
||||||
dur := optional.ToDuration(dm.DefaultTableExpiration)
|
|
||||||
if dur == 0 {
|
|
||||||
// Send a null to delete the field.
|
|
||||||
ds.NullFields = append(ds.NullFields, "DefaultTableExpirationMs")
|
|
||||||
} else {
|
|
||||||
ds.DefaultTableExpirationMs = int64(dur / time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if dm.Access != nil {
|
|
||||||
var err error
|
|
||||||
ds.Access, err = accessListToBQ(dm.Access)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(ds.Access) == 0 {
|
|
||||||
ds.NullFields = append(ds.NullFields, "Access")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
labels, forces, nulls := dm.update()
|
|
||||||
ds.Labels = labels
|
|
||||||
ds.ForceSendFields = append(ds.ForceSendFields, forces...)
|
|
||||||
ds.NullFields = append(ds.NullFields, nulls...)
|
|
||||||
return ds, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Table creates a handle to a BigQuery table in the dataset.
|
|
||||||
// To determine if a table exists, call Table.Metadata.
|
|
||||||
// If the table does not already exist, use Table.Create to create it.
|
|
||||||
func (d *Dataset) Table(tableID string) *Table {
|
|
||||||
return &Table{ProjectID: d.ProjectID, DatasetID: d.DatasetID, TableID: tableID, c: d.c}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tables returns an iterator over the tables in the Dataset.
|
|
||||||
func (d *Dataset) Tables(ctx context.Context) *TableIterator {
|
|
||||||
it := &TableIterator{
|
|
||||||
ctx: ctx,
|
|
||||||
dataset: d,
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
|
||||||
it.fetch,
|
|
||||||
func() int { return len(it.tables) },
|
|
||||||
func() interface{} { b := it.tables; it.tables = nil; return b })
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// A TableIterator is an iterator over Tables.
|
|
||||||
type TableIterator struct {
|
|
||||||
ctx context.Context
|
|
||||||
dataset *Dataset
|
|
||||||
tables []*Table
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next returns the next result. Its second return value is Done if there are
|
|
||||||
// no more results. Once Next returns Done, all subsequent calls will return
|
|
||||||
// Done.
|
|
||||||
func (it *TableIterator) Next() (*Table, error) {
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
t := it.tables[0]
|
|
||||||
it.tables = it.tables[1:]
|
|
||||||
return t, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *TableIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
|
||||||
|
|
||||||
// for testing
|
|
||||||
var listTables = func(it *TableIterator, pageSize int, pageToken string) (*bq.TableList, error) {
|
|
||||||
call := it.dataset.c.bqs.Tables.List(it.dataset.ProjectID, it.dataset.DatasetID).
|
|
||||||
PageToken(pageToken).
|
|
||||||
Context(it.ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if pageSize > 0 {
|
|
||||||
call.MaxResults(int64(pageSize))
|
|
||||||
}
|
|
||||||
var res *bq.TableList
|
|
||||||
err := runWithRetry(it.ctx, func() (err error) {
|
|
||||||
res, err = call.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TableIterator) fetch(pageSize int, pageToken string) (string, error) {
|
|
||||||
res, err := listTables(it, pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, t := range res.Tables {
|
|
||||||
it.tables = append(it.tables, bqToTable(t.TableReference, it.dataset.c))
|
|
||||||
}
|
|
||||||
return res.NextPageToken, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToTable(tr *bq.TableReference, c *Client) *Table {
|
|
||||||
if tr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &Table{
|
|
||||||
ProjectID: tr.ProjectId,
|
|
||||||
DatasetID: tr.DatasetId,
|
|
||||||
TableID: tr.TableId,
|
|
||||||
c: c,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Datasets returns an iterator over the datasets in a project.
|
|
||||||
// The Client's project is used by default, but that can be
|
|
||||||
// changed by setting ProjectID on the returned iterator before calling Next.
|
|
||||||
func (c *Client) Datasets(ctx context.Context) *DatasetIterator {
|
|
||||||
return c.DatasetsInProject(ctx, c.projectID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetsInProject returns an iterator over the datasets in the provided project.
|
|
||||||
//
|
|
||||||
// Deprecated: call Client.Datasets, then set ProjectID on the returned iterator.
|
|
||||||
func (c *Client) DatasetsInProject(ctx context.Context, projectID string) *DatasetIterator {
|
|
||||||
it := &DatasetIterator{
|
|
||||||
ctx: ctx,
|
|
||||||
c: c,
|
|
||||||
ProjectID: projectID,
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
|
||||||
it.fetch,
|
|
||||||
func() int { return len(it.items) },
|
|
||||||
func() interface{} { b := it.items; it.items = nil; return b })
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetIterator iterates over the datasets in a project.
|
|
||||||
type DatasetIterator struct {
|
|
||||||
// ListHidden causes hidden datasets to be listed when set to true.
|
|
||||||
// Set before the first call to Next.
|
|
||||||
ListHidden bool
|
|
||||||
|
|
||||||
// Filter restricts the datasets returned by label. The filter syntax is described in
|
|
||||||
// https://cloud.google.com/bigquery/docs/labeling-datasets#filtering_datasets_using_labels
|
|
||||||
// Set before the first call to Next.
|
|
||||||
Filter string
|
|
||||||
|
|
||||||
// The project ID of the listed datasets.
|
|
||||||
// Set before the first call to Next.
|
|
||||||
ProjectID string
|
|
||||||
|
|
||||||
ctx context.Context
|
|
||||||
c *Client
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
items []*Dataset
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *DatasetIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
|
||||||
|
|
||||||
func (it *DatasetIterator) Next() (*Dataset, error) {
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
item := it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// for testing
|
|
||||||
var listDatasets = func(it *DatasetIterator, pageSize int, pageToken string) (*bq.DatasetList, error) {
|
|
||||||
call := it.c.bqs.Datasets.List(it.ProjectID).
|
|
||||||
Context(it.ctx).
|
|
||||||
PageToken(pageToken).
|
|
||||||
All(it.ListHidden)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if pageSize > 0 {
|
|
||||||
call.MaxResults(int64(pageSize))
|
|
||||||
}
|
|
||||||
if it.Filter != "" {
|
|
||||||
call.Filter(it.Filter)
|
|
||||||
}
|
|
||||||
var res *bq.DatasetList
|
|
||||||
err := runWithRetry(it.ctx, func() (err error) {
|
|
||||||
res, err = call.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *DatasetIterator) fetch(pageSize int, pageToken string) (string, error) {
|
|
||||||
res, err := listDatasets(it, pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, d := range res.Datasets {
|
|
||||||
it.items = append(it.items, &Dataset{
|
|
||||||
ProjectID: d.DatasetReference.ProjectId,
|
|
||||||
DatasetID: d.DatasetReference.DatasetId,
|
|
||||||
c: it.c,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return res.NextPageToken, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// An AccessEntry describes the permissions that an entity has on a dataset.
|
|
||||||
type AccessEntry struct {
|
|
||||||
Role AccessRole // The role of the entity
|
|
||||||
EntityType EntityType // The type of entity
|
|
||||||
Entity string // The entity (individual or group) granted access
|
|
||||||
View *Table // The view granted access (EntityType must be ViewEntity)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AccessRole is the level of access to grant to a dataset.
|
|
||||||
type AccessRole string
|
|
||||||
|
|
||||||
const (
|
|
||||||
OwnerRole AccessRole = "OWNER"
|
|
||||||
ReaderRole AccessRole = "READER"
|
|
||||||
WriterRole AccessRole = "WRITER"
|
|
||||||
)
|
|
||||||
|
|
||||||
// EntityType is the type of entity in an AccessEntry.
|
|
||||||
type EntityType int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// A domain (e.g. "example.com")
|
|
||||||
DomainEntity EntityType = iota + 1
|
|
||||||
|
|
||||||
// Email address of a Google Group
|
|
||||||
GroupEmailEntity
|
|
||||||
|
|
||||||
// Email address of an individual user.
|
|
||||||
UserEmailEntity
|
|
||||||
|
|
||||||
// A special group: one of projectOwners, projectReaders, projectWriters or allAuthenticatedUsers.
|
|
||||||
SpecialGroupEntity
|
|
||||||
|
|
||||||
// A BigQuery view.
|
|
||||||
ViewEntity
|
|
||||||
)
|
|
||||||
|
|
||||||
func (e *AccessEntry) toBQ() (*bq.DatasetAccess, error) {
|
|
||||||
q := &bq.DatasetAccess{Role: string(e.Role)}
|
|
||||||
switch e.EntityType {
|
|
||||||
case DomainEntity:
|
|
||||||
q.Domain = e.Entity
|
|
||||||
case GroupEmailEntity:
|
|
||||||
q.GroupByEmail = e.Entity
|
|
||||||
case UserEmailEntity:
|
|
||||||
q.UserByEmail = e.Entity
|
|
||||||
case SpecialGroupEntity:
|
|
||||||
q.SpecialGroup = e.Entity
|
|
||||||
case ViewEntity:
|
|
||||||
q.View = e.View.toBQ()
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("bigquery: unknown entity type %d", e.EntityType)
|
|
||||||
}
|
|
||||||
return q, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToAccessEntry(q *bq.DatasetAccess, c *Client) (*AccessEntry, error) {
|
|
||||||
e := &AccessEntry{Role: AccessRole(q.Role)}
|
|
||||||
switch {
|
|
||||||
case q.Domain != "":
|
|
||||||
e.Entity = q.Domain
|
|
||||||
e.EntityType = DomainEntity
|
|
||||||
case q.GroupByEmail != "":
|
|
||||||
e.Entity = q.GroupByEmail
|
|
||||||
e.EntityType = GroupEmailEntity
|
|
||||||
case q.UserByEmail != "":
|
|
||||||
e.Entity = q.UserByEmail
|
|
||||||
e.EntityType = UserEmailEntity
|
|
||||||
case q.SpecialGroup != "":
|
|
||||||
e.Entity = q.SpecialGroup
|
|
||||||
e.EntityType = SpecialGroupEntity
|
|
||||||
case q.View != nil:
|
|
||||||
e.View = c.DatasetInProject(q.View.ProjectId, q.View.DatasetId).Table(q.View.TableId)
|
|
||||||
e.EntityType = ViewEntity
|
|
||||||
default:
|
|
||||||
return nil, errors.New("bigquery: invalid access value")
|
|
||||||
}
|
|
||||||
return e, nil
|
|
||||||
}
|
|
||||||
328
vendor/cloud.google.com/go/bigquery/dataset_test.go
generated
vendored
328
vendor/cloud.google.com/go/bigquery/dataset_test.go
generated
vendored
@ -1,328 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
itest "google.golang.org/api/iterator/testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// readServiceStub services read requests by returning data from an in-memory list of values.
|
|
||||||
type listTablesStub struct {
|
|
||||||
expectedProject, expectedDataset string
|
|
||||||
tables []*bq.TableListTables
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *listTablesStub) listTables(it *TableIterator, pageSize int, pageToken string) (*bq.TableList, error) {
|
|
||||||
if it.dataset.ProjectID != s.expectedProject {
|
|
||||||
return nil, errors.New("wrong project id")
|
|
||||||
}
|
|
||||||
if it.dataset.DatasetID != s.expectedDataset {
|
|
||||||
return nil, errors.New("wrong dataset id")
|
|
||||||
}
|
|
||||||
const maxPageSize = 2
|
|
||||||
if pageSize <= 0 || pageSize > maxPageSize {
|
|
||||||
pageSize = maxPageSize
|
|
||||||
}
|
|
||||||
start := 0
|
|
||||||
if pageToken != "" {
|
|
||||||
var err error
|
|
||||||
start, err = strconv.Atoi(pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end := start + pageSize
|
|
||||||
if end > len(s.tables) {
|
|
||||||
end = len(s.tables)
|
|
||||||
}
|
|
||||||
nextPageToken := ""
|
|
||||||
if end < len(s.tables) {
|
|
||||||
nextPageToken = strconv.Itoa(end)
|
|
||||||
}
|
|
||||||
return &bq.TableList{
|
|
||||||
Tables: s.tables[start:end],
|
|
||||||
NextPageToken: nextPageToken,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTables(t *testing.T) {
|
|
||||||
c := &Client{projectID: "p1"}
|
|
||||||
inTables := []*bq.TableListTables{
|
|
||||||
{TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t1"}},
|
|
||||||
{TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t2"}},
|
|
||||||
{TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t3"}},
|
|
||||||
}
|
|
||||||
outTables := []*Table{
|
|
||||||
{ProjectID: "p1", DatasetID: "d1", TableID: "t1", c: c},
|
|
||||||
{ProjectID: "p1", DatasetID: "d1", TableID: "t2", c: c},
|
|
||||||
{ProjectID: "p1", DatasetID: "d1", TableID: "t3", c: c},
|
|
||||||
}
|
|
||||||
|
|
||||||
lts := &listTablesStub{
|
|
||||||
expectedProject: "p1",
|
|
||||||
expectedDataset: "d1",
|
|
||||||
tables: inTables,
|
|
||||||
}
|
|
||||||
old := listTables
|
|
||||||
listTables = lts.listTables // cannot use t.Parallel with this test
|
|
||||||
defer func() { listTables = old }()
|
|
||||||
|
|
||||||
msg, ok := itest.TestIterator(outTables,
|
|
||||||
func() interface{} { return c.Dataset("d1").Tables(context.Background()) },
|
|
||||||
func(it interface{}) (interface{}, error) { return it.(*TableIterator).Next() })
|
|
||||||
if !ok {
|
|
||||||
t.Error(msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type listDatasetsStub struct {
|
|
||||||
expectedProject string
|
|
||||||
datasets []*bq.DatasetListDatasets
|
|
||||||
hidden map[*bq.DatasetListDatasets]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *listDatasetsStub) listDatasets(it *DatasetIterator, pageSize int, pageToken string) (*bq.DatasetList, error) {
|
|
||||||
const maxPageSize = 2
|
|
||||||
if pageSize <= 0 || pageSize > maxPageSize {
|
|
||||||
pageSize = maxPageSize
|
|
||||||
}
|
|
||||||
if it.Filter != "" {
|
|
||||||
return nil, errors.New("filter not supported")
|
|
||||||
}
|
|
||||||
if it.ProjectID != s.expectedProject {
|
|
||||||
return nil, errors.New("bad project ID")
|
|
||||||
}
|
|
||||||
start := 0
|
|
||||||
if pageToken != "" {
|
|
||||||
var err error
|
|
||||||
start, err = strconv.Atoi(pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var (
|
|
||||||
i int
|
|
||||||
result []*bq.DatasetListDatasets
|
|
||||||
nextPageToken string
|
|
||||||
)
|
|
||||||
for i = start; len(result) < pageSize && i < len(s.datasets); i++ {
|
|
||||||
if s.hidden[s.datasets[i]] && !it.ListHidden {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
result = append(result, s.datasets[i])
|
|
||||||
}
|
|
||||||
if i < len(s.datasets) {
|
|
||||||
nextPageToken = strconv.Itoa(i)
|
|
||||||
}
|
|
||||||
return &bq.DatasetList{
|
|
||||||
Datasets: result,
|
|
||||||
NextPageToken: nextPageToken,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasets(t *testing.T) {
|
|
||||||
client := &Client{projectID: "p"}
|
|
||||||
inDatasets := []*bq.DatasetListDatasets{
|
|
||||||
{DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "a"}},
|
|
||||||
{DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "b"}},
|
|
||||||
{DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "hidden"}},
|
|
||||||
{DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "c"}},
|
|
||||||
}
|
|
||||||
outDatasets := []*Dataset{
|
|
||||||
{"p", "a", client},
|
|
||||||
{"p", "b", client},
|
|
||||||
{"p", "hidden", client},
|
|
||||||
{"p", "c", client},
|
|
||||||
}
|
|
||||||
lds := &listDatasetsStub{
|
|
||||||
expectedProject: "p",
|
|
||||||
datasets: inDatasets,
|
|
||||||
hidden: map[*bq.DatasetListDatasets]bool{inDatasets[2]: true},
|
|
||||||
}
|
|
||||||
old := listDatasets
|
|
||||||
listDatasets = lds.listDatasets // cannot use t.Parallel with this test
|
|
||||||
defer func() { listDatasets = old }()
|
|
||||||
|
|
||||||
msg, ok := itest.TestIterator(outDatasets,
|
|
||||||
func() interface{} { it := client.Datasets(context.Background()); it.ListHidden = true; return it },
|
|
||||||
func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("ListHidden=true: %s", msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
msg, ok = itest.TestIterator([]*Dataset{outDatasets[0], outDatasets[1], outDatasets[3]},
|
|
||||||
func() interface{} { it := client.Datasets(context.Background()); it.ListHidden = false; return it },
|
|
||||||
func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("ListHidden=false: %s", msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetToBQ(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
in *DatasetMetadata
|
|
||||||
want *bq.Dataset
|
|
||||||
}{
|
|
||||||
{nil, &bq.Dataset{}},
|
|
||||||
{&DatasetMetadata{Name: "name"}, &bq.Dataset{FriendlyName: "name"}},
|
|
||||||
{&DatasetMetadata{
|
|
||||||
Name: "name",
|
|
||||||
Description: "desc",
|
|
||||||
DefaultTableExpiration: time.Hour,
|
|
||||||
Location: "EU",
|
|
||||||
Labels: map[string]string{"x": "y"},
|
|
||||||
Access: []*AccessEntry{{Role: OwnerRole, Entity: "example.com", EntityType: DomainEntity}},
|
|
||||||
}, &bq.Dataset{
|
|
||||||
FriendlyName: "name",
|
|
||||||
Description: "desc",
|
|
||||||
DefaultTableExpirationMs: 60 * 60 * 1000,
|
|
||||||
Location: "EU",
|
|
||||||
Labels: map[string]string{"x": "y"},
|
|
||||||
Access: []*bq.DatasetAccess{{Role: "OWNER", Domain: "example.com"}},
|
|
||||||
}},
|
|
||||||
} {
|
|
||||||
got, err := test.in.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%v:\ngot %+v\nwant %+v", test.in, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check that non-writeable fields are unset.
|
|
||||||
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
||||||
for _, dm := range []*DatasetMetadata{
|
|
||||||
{CreationTime: aTime},
|
|
||||||
{LastModifiedTime: aTime},
|
|
||||||
{FullID: "x"},
|
|
||||||
{ETag: "e"},
|
|
||||||
} {
|
|
||||||
if _, err := dm.toBQ(); err == nil {
|
|
||||||
t.Errorf("%+v: got nil, want error", dm)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBQToDatasetMetadata(t *testing.T) {
|
|
||||||
cTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
||||||
cMillis := cTime.UnixNano() / 1e6
|
|
||||||
mTime := time.Date(2017, 10, 31, 0, 0, 0, 0, time.Local)
|
|
||||||
mMillis := mTime.UnixNano() / 1e6
|
|
||||||
q := &bq.Dataset{
|
|
||||||
CreationTime: cMillis,
|
|
||||||
LastModifiedTime: mMillis,
|
|
||||||
FriendlyName: "name",
|
|
||||||
Description: "desc",
|
|
||||||
DefaultTableExpirationMs: 60 * 60 * 1000,
|
|
||||||
Location: "EU",
|
|
||||||
Labels: map[string]string{"x": "y"},
|
|
||||||
Access: []*bq.DatasetAccess{
|
|
||||||
{Role: "READER", UserByEmail: "joe@example.com"},
|
|
||||||
{Role: "WRITER", GroupByEmail: "users@example.com"},
|
|
||||||
},
|
|
||||||
Etag: "etag",
|
|
||||||
}
|
|
||||||
want := &DatasetMetadata{
|
|
||||||
CreationTime: cTime,
|
|
||||||
LastModifiedTime: mTime,
|
|
||||||
Name: "name",
|
|
||||||
Description: "desc",
|
|
||||||
DefaultTableExpiration: time.Hour,
|
|
||||||
Location: "EU",
|
|
||||||
Labels: map[string]string{"x": "y"},
|
|
||||||
Access: []*AccessEntry{
|
|
||||||
{Role: ReaderRole, Entity: "joe@example.com", EntityType: UserEmailEntity},
|
|
||||||
{Role: WriterRole, Entity: "users@example.com", EntityType: GroupEmailEntity},
|
|
||||||
},
|
|
||||||
ETag: "etag",
|
|
||||||
}
|
|
||||||
got, err := bqToDatasetMetadata(q)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Errorf("-got, +want:\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
|
|
||||||
dm := DatasetMetadataToUpdate{
|
|
||||||
Description: "desc",
|
|
||||||
Name: "name",
|
|
||||||
DefaultTableExpiration: time.Hour,
|
|
||||||
}
|
|
||||||
dm.SetLabel("label", "value")
|
|
||||||
dm.DeleteLabel("del")
|
|
||||||
|
|
||||||
got, err := dm.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := &bq.Dataset{
|
|
||||||
Description: "desc",
|
|
||||||
FriendlyName: "name",
|
|
||||||
DefaultTableExpirationMs: 60 * 60 * 1000,
|
|
||||||
Labels: map[string]string{"label": "value"},
|
|
||||||
ForceSendFields: []string{"Description", "FriendlyName"},
|
|
||||||
NullFields: []string{"Labels.del"},
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Errorf("-got, +want:\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConvertAccessEntry(t *testing.T) {
|
|
||||||
c := &Client{projectID: "pid"}
|
|
||||||
for _, e := range []*AccessEntry{
|
|
||||||
{Role: ReaderRole, Entity: "e", EntityType: DomainEntity},
|
|
||||||
{Role: WriterRole, Entity: "e", EntityType: GroupEmailEntity},
|
|
||||||
{Role: OwnerRole, Entity: "e", EntityType: UserEmailEntity},
|
|
||||||
{Role: ReaderRole, Entity: "e", EntityType: SpecialGroupEntity},
|
|
||||||
{Role: ReaderRole, EntityType: ViewEntity,
|
|
||||||
View: &Table{ProjectID: "p", DatasetID: "d", TableID: "t", c: c}},
|
|
||||||
} {
|
|
||||||
q, err := e.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, err := bqToAccessEntry(q, c)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, e, cmp.AllowUnexported(Table{}, Client{})); diff != "" {
|
|
||||||
t.Errorf("got=-, want=+:\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
e := &AccessEntry{Role: ReaderRole, Entity: "e"}
|
|
||||||
if _, err := e.toBQ(); err == nil {
|
|
||||||
t.Error("got nil, want error")
|
|
||||||
}
|
|
||||||
if _, err := bqToAccessEntry(&bq.DatasetAccess{Role: "WRITER"}, nil); err == nil {
|
|
||||||
t.Error("got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
67
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/ListDataSources_smoke_test.go
generated
vendored
67
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/ListDataSources_smoke_test.go
generated
vendored
@ -1,67 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// AUTO-GENERATED CODE. DO NOT EDIT.
|
|
||||||
|
|
||||||
package datatransfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
datatransferpb "google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1"
|
|
||||||
)
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ = fmt.Sprintf
|
|
||||||
var _ = iterator.Done
|
|
||||||
var _ = strconv.FormatUint
|
|
||||||
var _ = time.Now
|
|
||||||
|
|
||||||
func TestDataTransferServiceSmoke(t *testing.T) {
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("skipping smoke test in short mode")
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
ts := testutil.TokenSource(ctx, DefaultAuthScopes()...)
|
|
||||||
if ts == nil {
|
|
||||||
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
|
|
||||||
}
|
|
||||||
|
|
||||||
projectId := testutil.ProjID()
|
|
||||||
_ = projectId
|
|
||||||
|
|
||||||
c, err := NewClient(ctx, option.WithTokenSource(ts))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var formattedParent string = fmt.Sprintf("projects/%s", projectId)
|
|
||||||
var request = &datatransferpb.ListDataSourcesRequest{
|
|
||||||
Parent: formattedParent,
|
|
||||||
}
|
|
||||||
|
|
||||||
iter := c.ListDataSources(ctx, request)
|
|
||||||
if _, err := iter.Next(); err != nil && err != iterator.Done {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
603
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/data_transfer_client.go
generated
vendored
603
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/data_transfer_client.go
generated
vendored
@ -1,603 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// AUTO-GENERATED CODE. DO NOT EDIT.
|
|
||||||
|
|
||||||
package datatransfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/version"
|
|
||||||
gax "github.com/googleapis/gax-go"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
"google.golang.org/api/transport"
|
|
||||||
datatransferpb "google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1"
|
|
||||||
"google.golang.org/grpc"
|
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
"google.golang.org/grpc/metadata"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CallOptions contains the retry settings for each method of Client.
|
|
||||||
type CallOptions struct {
|
|
||||||
GetDataSource []gax.CallOption
|
|
||||||
ListDataSources []gax.CallOption
|
|
||||||
CreateTransferConfig []gax.CallOption
|
|
||||||
UpdateTransferConfig []gax.CallOption
|
|
||||||
DeleteTransferConfig []gax.CallOption
|
|
||||||
GetTransferConfig []gax.CallOption
|
|
||||||
ListTransferConfigs []gax.CallOption
|
|
||||||
ScheduleTransferRuns []gax.CallOption
|
|
||||||
GetTransferRun []gax.CallOption
|
|
||||||
DeleteTransferRun []gax.CallOption
|
|
||||||
ListTransferRuns []gax.CallOption
|
|
||||||
ListTransferLogs []gax.CallOption
|
|
||||||
CheckValidCreds []gax.CallOption
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultClientOptions() []option.ClientOption {
|
|
||||||
return []option.ClientOption{
|
|
||||||
option.WithEndpoint("bigquerydatatransfer.googleapis.com:443"),
|
|
||||||
option.WithScopes(DefaultAuthScopes()...),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultCallOptions() *CallOptions {
|
|
||||||
retry := map[[2]string][]gax.CallOption{
|
|
||||||
{"default", "idempotent"}: {
|
|
||||||
gax.WithRetry(func() gax.Retryer {
|
|
||||||
return gax.OnCodes([]codes.Code{
|
|
||||||
codes.DeadlineExceeded,
|
|
||||||
codes.Unavailable,
|
|
||||||
}, gax.Backoff{
|
|
||||||
Initial: 100 * time.Millisecond,
|
|
||||||
Max: 60000 * time.Millisecond,
|
|
||||||
Multiplier: 1.3,
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return &CallOptions{
|
|
||||||
GetDataSource: retry[[2]string{"default", "idempotent"}],
|
|
||||||
ListDataSources: retry[[2]string{"default", "idempotent"}],
|
|
||||||
CreateTransferConfig: retry[[2]string{"default", "non_idempotent"}],
|
|
||||||
UpdateTransferConfig: retry[[2]string{"default", "non_idempotent"}],
|
|
||||||
DeleteTransferConfig: retry[[2]string{"default", "idempotent"}],
|
|
||||||
GetTransferConfig: retry[[2]string{"default", "idempotent"}],
|
|
||||||
ListTransferConfigs: retry[[2]string{"default", "idempotent"}],
|
|
||||||
ScheduleTransferRuns: retry[[2]string{"default", "non_idempotent"}],
|
|
||||||
GetTransferRun: retry[[2]string{"default", "idempotent"}],
|
|
||||||
DeleteTransferRun: retry[[2]string{"default", "idempotent"}],
|
|
||||||
ListTransferRuns: retry[[2]string{"default", "idempotent"}],
|
|
||||||
ListTransferLogs: retry[[2]string{"default", "idempotent"}],
|
|
||||||
CheckValidCreds: retry[[2]string{"default", "idempotent"}],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client is a client for interacting with BigQuery Data Transfer API.
|
|
||||||
//
|
|
||||||
// Methods, except Close, may be called concurrently. However, fields must not be modified concurrently with method calls.
|
|
||||||
type Client struct {
|
|
||||||
// The connection to the service.
|
|
||||||
conn *grpc.ClientConn
|
|
||||||
|
|
||||||
// The gRPC API client.
|
|
||||||
client datatransferpb.DataTransferServiceClient
|
|
||||||
|
|
||||||
// The call options for this service.
|
|
||||||
CallOptions *CallOptions
|
|
||||||
|
|
||||||
// The x-goog-* metadata to be sent with each request.
|
|
||||||
xGoogMetadata metadata.MD
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient creates a new data transfer service client.
|
|
||||||
//
|
|
||||||
// The Google BigQuery Data Transfer Service API enables BigQuery users to
|
|
||||||
// configure the transfer of their data from other Google Products into BigQuery.
|
|
||||||
// This service contains methods that are end user exposed. It backs up the
|
|
||||||
// frontend.
|
|
||||||
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
|
|
||||||
conn, err := transport.DialGRPC(ctx, append(defaultClientOptions(), opts...)...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
c := &Client{
|
|
||||||
conn: conn,
|
|
||||||
CallOptions: defaultCallOptions(),
|
|
||||||
|
|
||||||
client: datatransferpb.NewDataTransferServiceClient(conn),
|
|
||||||
}
|
|
||||||
c.setGoogleClientInfo()
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connection returns the client's connection to the API service.
|
|
||||||
func (c *Client) Connection() *grpc.ClientConn {
|
|
||||||
return c.conn
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the connection to the API service. The user should invoke this when
|
|
||||||
// the client is no longer required.
|
|
||||||
func (c *Client) Close() error {
|
|
||||||
return c.conn.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// setGoogleClientInfo sets the name and version of the application in
|
|
||||||
// the `x-goog-api-client` header passed on each request. Intended for
|
|
||||||
// use by Google-written clients.
|
|
||||||
func (c *Client) setGoogleClientInfo(keyval ...string) {
|
|
||||||
kv := append([]string{"gl-go", version.Go()}, keyval...)
|
|
||||||
kv = append(kv, "gapic", version.Repo, "gax", gax.Version, "grpc", grpc.Version)
|
|
||||||
c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetDataSource retrieves a supported data source and returns its settings,
|
|
||||||
// which can be used for UI rendering.
|
|
||||||
func (c *Client) GetDataSource(ctx context.Context, req *datatransferpb.GetDataSourceRequest, opts ...gax.CallOption) (*datatransferpb.DataSource, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.GetDataSource[0:len(c.CallOptions.GetDataSource):len(c.CallOptions.GetDataSource)], opts...)
|
|
||||||
var resp *datatransferpb.DataSource
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.GetDataSource(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDataSources lists supported data sources and returns their settings,
|
|
||||||
// which can be used for UI rendering.
|
|
||||||
func (c *Client) ListDataSources(ctx context.Context, req *datatransferpb.ListDataSourcesRequest, opts ...gax.CallOption) *DataSourceIterator {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.ListDataSources[0:len(c.CallOptions.ListDataSources):len(c.CallOptions.ListDataSources)], opts...)
|
|
||||||
it := &DataSourceIterator{}
|
|
||||||
it.InternalFetch = func(pageSize int, pageToken string) ([]*datatransferpb.DataSource, string, error) {
|
|
||||||
var resp *datatransferpb.ListDataSourcesResponse
|
|
||||||
req.PageToken = pageToken
|
|
||||||
if pageSize > math.MaxInt32 {
|
|
||||||
req.PageSize = math.MaxInt32
|
|
||||||
} else {
|
|
||||||
req.PageSize = int32(pageSize)
|
|
||||||
}
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.ListDataSources(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return resp.DataSources, resp.NextPageToken, nil
|
|
||||||
}
|
|
||||||
fetch := func(pageSize int, pageToken string) (string, error) {
|
|
||||||
items, nextPageToken, err := it.InternalFetch(pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.items = append(it.items, items...)
|
|
||||||
return nextPageToken, nil
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf)
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateTransferConfig creates a new data transfer configuration.
|
|
||||||
func (c *Client) CreateTransferConfig(ctx context.Context, req *datatransferpb.CreateTransferConfigRequest, opts ...gax.CallOption) (*datatransferpb.TransferConfig, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.CreateTransferConfig[0:len(c.CallOptions.CreateTransferConfig):len(c.CallOptions.CreateTransferConfig)], opts...)
|
|
||||||
var resp *datatransferpb.TransferConfig
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.CreateTransferConfig(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateTransferConfig updates a data transfer configuration.
|
|
||||||
// All fields must be set, even if they are not updated.
|
|
||||||
func (c *Client) UpdateTransferConfig(ctx context.Context, req *datatransferpb.UpdateTransferConfigRequest, opts ...gax.CallOption) (*datatransferpb.TransferConfig, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.UpdateTransferConfig[0:len(c.CallOptions.UpdateTransferConfig):len(c.CallOptions.UpdateTransferConfig)], opts...)
|
|
||||||
var resp *datatransferpb.TransferConfig
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.UpdateTransferConfig(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteTransferConfig deletes a data transfer configuration,
|
|
||||||
// including any associated transfer runs and logs.
|
|
||||||
func (c *Client) DeleteTransferConfig(ctx context.Context, req *datatransferpb.DeleteTransferConfigRequest, opts ...gax.CallOption) error {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.DeleteTransferConfig[0:len(c.CallOptions.DeleteTransferConfig):len(c.CallOptions.DeleteTransferConfig)], opts...)
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
_, err = c.client.DeleteTransferConfig(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetTransferConfig returns information about a data transfer config.
|
|
||||||
func (c *Client) GetTransferConfig(ctx context.Context, req *datatransferpb.GetTransferConfigRequest, opts ...gax.CallOption) (*datatransferpb.TransferConfig, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.GetTransferConfig[0:len(c.CallOptions.GetTransferConfig):len(c.CallOptions.GetTransferConfig)], opts...)
|
|
||||||
var resp *datatransferpb.TransferConfig
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.GetTransferConfig(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListTransferConfigs returns information about all data transfers in the project.
|
|
||||||
func (c *Client) ListTransferConfigs(ctx context.Context, req *datatransferpb.ListTransferConfigsRequest, opts ...gax.CallOption) *TransferConfigIterator {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.ListTransferConfigs[0:len(c.CallOptions.ListTransferConfigs):len(c.CallOptions.ListTransferConfigs)], opts...)
|
|
||||||
it := &TransferConfigIterator{}
|
|
||||||
it.InternalFetch = func(pageSize int, pageToken string) ([]*datatransferpb.TransferConfig, string, error) {
|
|
||||||
var resp *datatransferpb.ListTransferConfigsResponse
|
|
||||||
req.PageToken = pageToken
|
|
||||||
if pageSize > math.MaxInt32 {
|
|
||||||
req.PageSize = math.MaxInt32
|
|
||||||
} else {
|
|
||||||
req.PageSize = int32(pageSize)
|
|
||||||
}
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.ListTransferConfigs(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return resp.TransferConfigs, resp.NextPageToken, nil
|
|
||||||
}
|
|
||||||
fetch := func(pageSize int, pageToken string) (string, error) {
|
|
||||||
items, nextPageToken, err := it.InternalFetch(pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.items = append(it.items, items...)
|
|
||||||
return nextPageToken, nil
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf)
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// ScheduleTransferRuns creates transfer runs for a time range [start_time, end_time].
|
|
||||||
// For each date - or whatever granularity the data source supports - in the
|
|
||||||
// range, one transfer run is created.
|
|
||||||
// Note that runs are created per UTC time in the time range.
|
|
||||||
func (c *Client) ScheduleTransferRuns(ctx context.Context, req *datatransferpb.ScheduleTransferRunsRequest, opts ...gax.CallOption) (*datatransferpb.ScheduleTransferRunsResponse, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.ScheduleTransferRuns[0:len(c.CallOptions.ScheduleTransferRuns):len(c.CallOptions.ScheduleTransferRuns)], opts...)
|
|
||||||
var resp *datatransferpb.ScheduleTransferRunsResponse
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.ScheduleTransferRuns(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetTransferRun returns information about the particular transfer run.
|
|
||||||
func (c *Client) GetTransferRun(ctx context.Context, req *datatransferpb.GetTransferRunRequest, opts ...gax.CallOption) (*datatransferpb.TransferRun, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.GetTransferRun[0:len(c.CallOptions.GetTransferRun):len(c.CallOptions.GetTransferRun)], opts...)
|
|
||||||
var resp *datatransferpb.TransferRun
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.GetTransferRun(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteTransferRun deletes the specified transfer run.
|
|
||||||
func (c *Client) DeleteTransferRun(ctx context.Context, req *datatransferpb.DeleteTransferRunRequest, opts ...gax.CallOption) error {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.DeleteTransferRun[0:len(c.CallOptions.DeleteTransferRun):len(c.CallOptions.DeleteTransferRun)], opts...)
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
_, err = c.client.DeleteTransferRun(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListTransferRuns returns information about running and completed jobs.
|
|
||||||
func (c *Client) ListTransferRuns(ctx context.Context, req *datatransferpb.ListTransferRunsRequest, opts ...gax.CallOption) *TransferRunIterator {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.ListTransferRuns[0:len(c.CallOptions.ListTransferRuns):len(c.CallOptions.ListTransferRuns)], opts...)
|
|
||||||
it := &TransferRunIterator{}
|
|
||||||
it.InternalFetch = func(pageSize int, pageToken string) ([]*datatransferpb.TransferRun, string, error) {
|
|
||||||
var resp *datatransferpb.ListTransferRunsResponse
|
|
||||||
req.PageToken = pageToken
|
|
||||||
if pageSize > math.MaxInt32 {
|
|
||||||
req.PageSize = math.MaxInt32
|
|
||||||
} else {
|
|
||||||
req.PageSize = int32(pageSize)
|
|
||||||
}
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.ListTransferRuns(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return resp.TransferRuns, resp.NextPageToken, nil
|
|
||||||
}
|
|
||||||
fetch := func(pageSize int, pageToken string) (string, error) {
|
|
||||||
items, nextPageToken, err := it.InternalFetch(pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.items = append(it.items, items...)
|
|
||||||
return nextPageToken, nil
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf)
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListTransferLogs returns user facing log messages for the data transfer run.
|
|
||||||
func (c *Client) ListTransferLogs(ctx context.Context, req *datatransferpb.ListTransferLogsRequest, opts ...gax.CallOption) *TransferMessageIterator {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.ListTransferLogs[0:len(c.CallOptions.ListTransferLogs):len(c.CallOptions.ListTransferLogs)], opts...)
|
|
||||||
it := &TransferMessageIterator{}
|
|
||||||
it.InternalFetch = func(pageSize int, pageToken string) ([]*datatransferpb.TransferMessage, string, error) {
|
|
||||||
var resp *datatransferpb.ListTransferLogsResponse
|
|
||||||
req.PageToken = pageToken
|
|
||||||
if pageSize > math.MaxInt32 {
|
|
||||||
req.PageSize = math.MaxInt32
|
|
||||||
} else {
|
|
||||||
req.PageSize = int32(pageSize)
|
|
||||||
}
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.ListTransferLogs(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return resp.TransferMessages, resp.NextPageToken, nil
|
|
||||||
}
|
|
||||||
fetch := func(pageSize int, pageToken string) (string, error) {
|
|
||||||
items, nextPageToken, err := it.InternalFetch(pageSize, pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.items = append(it.items, items...)
|
|
||||||
return nextPageToken, nil
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf)
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckValidCreds returns true if valid credentials exist for the given data source and
|
|
||||||
// requesting user.
|
|
||||||
// Some data sources doesn't support service account, so we need to talk to
|
|
||||||
// them on behalf of the end user. This API just checks whether we have OAuth
|
|
||||||
// token for the particular user, which is a pre-requisite before user can
|
|
||||||
// create a transfer config.
|
|
||||||
func (c *Client) CheckValidCreds(ctx context.Context, req *datatransferpb.CheckValidCredsRequest, opts ...gax.CallOption) (*datatransferpb.CheckValidCredsResponse, error) {
|
|
||||||
ctx = insertMetadata(ctx, c.xGoogMetadata)
|
|
||||||
opts = append(c.CallOptions.CheckValidCreds[0:len(c.CallOptions.CheckValidCreds):len(c.CallOptions.CheckValidCreds)], opts...)
|
|
||||||
var resp *datatransferpb.CheckValidCredsResponse
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
|
||||||
var err error
|
|
||||||
resp, err = c.client.CheckValidCreds(ctx, req, settings.GRPC...)
|
|
||||||
return err
|
|
||||||
}, opts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DataSourceIterator manages a stream of *datatransferpb.DataSource.
|
|
||||||
type DataSourceIterator struct {
|
|
||||||
items []*datatransferpb.DataSource
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
|
|
||||||
// InternalFetch is for use by the Google Cloud Libraries only.
|
|
||||||
// It is not part of the stable interface of this package.
|
|
||||||
//
|
|
||||||
// InternalFetch returns results from a single call to the underlying RPC.
|
|
||||||
// The number of results is no greater than pageSize.
|
|
||||||
// If there are no more results, nextPageToken is empty and err is nil.
|
|
||||||
InternalFetch func(pageSize int, pageToken string) (results []*datatransferpb.DataSource, nextPageToken string, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *DataSourceIterator) PageInfo() *iterator.PageInfo {
|
|
||||||
return it.pageInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next returns the next result. Its second return value is iterator.Done if there are no more
|
|
||||||
// results. Once Next returns Done, all subsequent calls will return Done.
|
|
||||||
func (it *DataSourceIterator) Next() (*datatransferpb.DataSource, error) {
|
|
||||||
var item *datatransferpb.DataSource
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return item, err
|
|
||||||
}
|
|
||||||
item = it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *DataSourceIterator) bufLen() int {
|
|
||||||
return len(it.items)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *DataSourceIterator) takeBuf() interface{} {
|
|
||||||
b := it.items
|
|
||||||
it.items = nil
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// TransferConfigIterator manages a stream of *datatransferpb.TransferConfig.
|
|
||||||
type TransferConfigIterator struct {
|
|
||||||
items []*datatransferpb.TransferConfig
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
|
|
||||||
// InternalFetch is for use by the Google Cloud Libraries only.
|
|
||||||
// It is not part of the stable interface of this package.
|
|
||||||
//
|
|
||||||
// InternalFetch returns results from a single call to the underlying RPC.
|
|
||||||
// The number of results is no greater than pageSize.
|
|
||||||
// If there are no more results, nextPageToken is empty and err is nil.
|
|
||||||
InternalFetch func(pageSize int, pageToken string) (results []*datatransferpb.TransferConfig, nextPageToken string, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *TransferConfigIterator) PageInfo() *iterator.PageInfo {
|
|
||||||
return it.pageInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next returns the next result. Its second return value is iterator.Done if there are no more
|
|
||||||
// results. Once Next returns Done, all subsequent calls will return Done.
|
|
||||||
func (it *TransferConfigIterator) Next() (*datatransferpb.TransferConfig, error) {
|
|
||||||
var item *datatransferpb.TransferConfig
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return item, err
|
|
||||||
}
|
|
||||||
item = it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferConfigIterator) bufLen() int {
|
|
||||||
return len(it.items)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferConfigIterator) takeBuf() interface{} {
|
|
||||||
b := it.items
|
|
||||||
it.items = nil
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// TransferMessageIterator manages a stream of *datatransferpb.TransferMessage.
|
|
||||||
type TransferMessageIterator struct {
|
|
||||||
items []*datatransferpb.TransferMessage
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
|
|
||||||
// InternalFetch is for use by the Google Cloud Libraries only.
|
|
||||||
// It is not part of the stable interface of this package.
|
|
||||||
//
|
|
||||||
// InternalFetch returns results from a single call to the underlying RPC.
|
|
||||||
// The number of results is no greater than pageSize.
|
|
||||||
// If there are no more results, nextPageToken is empty and err is nil.
|
|
||||||
InternalFetch func(pageSize int, pageToken string) (results []*datatransferpb.TransferMessage, nextPageToken string, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *TransferMessageIterator) PageInfo() *iterator.PageInfo {
|
|
||||||
return it.pageInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next returns the next result. Its second return value is iterator.Done if there are no more
|
|
||||||
// results. Once Next returns Done, all subsequent calls will return Done.
|
|
||||||
func (it *TransferMessageIterator) Next() (*datatransferpb.TransferMessage, error) {
|
|
||||||
var item *datatransferpb.TransferMessage
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return item, err
|
|
||||||
}
|
|
||||||
item = it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferMessageIterator) bufLen() int {
|
|
||||||
return len(it.items)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferMessageIterator) takeBuf() interface{} {
|
|
||||||
b := it.items
|
|
||||||
it.items = nil
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// TransferRunIterator manages a stream of *datatransferpb.TransferRun.
|
|
||||||
type TransferRunIterator struct {
|
|
||||||
items []*datatransferpb.TransferRun
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
|
|
||||||
// InternalFetch is for use by the Google Cloud Libraries only.
|
|
||||||
// It is not part of the stable interface of this package.
|
|
||||||
//
|
|
||||||
// InternalFetch returns results from a single call to the underlying RPC.
|
|
||||||
// The number of results is no greater than pageSize.
|
|
||||||
// If there are no more results, nextPageToken is empty and err is nil.
|
|
||||||
InternalFetch func(pageSize int, pageToken string) (results []*datatransferpb.TransferRun, nextPageToken string, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *TransferRunIterator) PageInfo() *iterator.PageInfo {
|
|
||||||
return it.pageInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next returns the next result. Its second return value is iterator.Done if there are no more
|
|
||||||
// results. Once Next returns Done, all subsequent calls will return Done.
|
|
||||||
func (it *TransferRunIterator) Next() (*datatransferpb.TransferRun, error) {
|
|
||||||
var item *datatransferpb.TransferRun
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return item, err
|
|
||||||
}
|
|
||||||
item = it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferRunIterator) bufLen() int {
|
|
||||||
return len(it.items)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *TransferRunIterator) takeBuf() interface{} {
|
|
||||||
b := it.items
|
|
||||||
it.items = nil
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
288
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/data_transfer_client_example_test.go
generated
vendored
288
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/data_transfer_client_example_test.go
generated
vendored
@ -1,288 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// AUTO-GENERATED CODE. DO NOT EDIT.
|
|
||||||
|
|
||||||
package datatransfer_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cloud.google.com/go/bigquery/datatransfer/apiv1"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
datatransferpb "google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ExampleNewClient() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use client.
|
|
||||||
_ = c
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_GetDataSource() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.GetDataSourceRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.GetDataSource(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_ListDataSources() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.ListDataSourcesRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
it := c.ListDataSources(ctx, req)
|
|
||||||
for {
|
|
||||||
resp, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_CreateTransferConfig() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.CreateTransferConfigRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.CreateTransferConfig(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_UpdateTransferConfig() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.UpdateTransferConfigRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.UpdateTransferConfig(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_DeleteTransferConfig() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.DeleteTransferConfigRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
err = c.DeleteTransferConfig(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_GetTransferConfig() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.GetTransferConfigRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.GetTransferConfig(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_ListTransferConfigs() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.ListTransferConfigsRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
it := c.ListTransferConfigs(ctx, req)
|
|
||||||
for {
|
|
||||||
resp, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_ScheduleTransferRuns() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.ScheduleTransferRunsRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.ScheduleTransferRuns(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_GetTransferRun() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.GetTransferRunRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.GetTransferRun(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_DeleteTransferRun() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.DeleteTransferRunRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
err = c.DeleteTransferRun(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_ListTransferRuns() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.ListTransferRunsRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
it := c.ListTransferRuns(ctx, req)
|
|
||||||
for {
|
|
||||||
resp, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_ListTransferLogs() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.ListTransferLogsRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
it := c.ListTransferLogs(ctx, req)
|
|
||||||
for {
|
|
||||||
resp, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_CheckValidCreds() {
|
|
||||||
ctx := context.Background()
|
|
||||||
c, err := datatransfer.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &datatransferpb.CheckValidCredsRequest{
|
|
||||||
// TODO: Fill request struct fields.
|
|
||||||
}
|
|
||||||
resp, err := c.CheckValidCreds(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// TODO: Use resp.
|
|
||||||
_ = resp
|
|
||||||
}
|
|
||||||
47
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/doc.go
generated
vendored
47
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/doc.go
generated
vendored
@ -1,47 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// AUTO-GENERATED CODE. DO NOT EDIT.
|
|
||||||
|
|
||||||
// Package datatransfer is an auto-generated package for the
|
|
||||||
// BigQuery Data Transfer API.
|
|
||||||
//
|
|
||||||
// NOTE: This package is in alpha. It is not stable, and is likely to change.
|
|
||||||
//
|
|
||||||
// Transfers data from partner SaaS applications to Google BigQuery on a
|
|
||||||
// scheduled, managed basis.
|
|
||||||
package datatransfer // import "cloud.google.com/go/bigquery/datatransfer/apiv1"
|
|
||||||
|
|
||||||
import (
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/grpc/metadata"
|
|
||||||
)
|
|
||||||
|
|
||||||
func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context {
|
|
||||||
out, _ := metadata.FromOutgoingContext(ctx)
|
|
||||||
out = out.Copy()
|
|
||||||
for _, md := range mds {
|
|
||||||
for k, v := range md {
|
|
||||||
out[k] = append(out[k], v...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return metadata.NewOutgoingContext(ctx, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultAuthScopes reports the default set of authentication scopes to use with this package.
|
|
||||||
func DefaultAuthScopes() []string {
|
|
||||||
return []string{
|
|
||||||
"https://www.googleapis.com/auth/cloud-platform",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
1146
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/mock_test.go
generated
vendored
1146
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/mock_test.go
generated
vendored
File diff suppressed because it is too large
Load Diff
135
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/path_funcs.go
generated
vendored
135
vendor/cloud.google.com/go/bigquery/datatransfer/apiv1/path_funcs.go
generated
vendored
@ -1,135 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// https://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package datatransfer
|
|
||||||
|
|
||||||
// ProjectPath returns the path for the project resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s", project)
|
|
||||||
// instead.
|
|
||||||
func ProjectPath(project string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// LocationPath returns the path for the location resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/locations/%s", project, location)
|
|
||||||
// instead.
|
|
||||||
func LocationPath(project, location string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/locations/" +
|
|
||||||
location +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// LocationDataSourcePath returns the path for the location data source resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/locations/%s/dataSources/%s", project, location, dataSource)
|
|
||||||
// instead.
|
|
||||||
func LocationDataSourcePath(project, location, dataSource string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/locations/" +
|
|
||||||
location +
|
|
||||||
"/dataSources/" +
|
|
||||||
dataSource +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// LocationTransferConfigPath returns the path for the location transfer config resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/locations/%s/transferConfigs/%s", project, location, transferConfig)
|
|
||||||
// instead.
|
|
||||||
func LocationTransferConfigPath(project, location, transferConfig string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/locations/" +
|
|
||||||
location +
|
|
||||||
"/transferConfigs/" +
|
|
||||||
transferConfig +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// LocationRunPath returns the path for the location run resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/locations/%s/transferConfigs/%s/runs/%s", project, location, transferConfig, run)
|
|
||||||
// instead.
|
|
||||||
func LocationRunPath(project, location, transferConfig, run string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/locations/" +
|
|
||||||
location +
|
|
||||||
"/transferConfigs/" +
|
|
||||||
transferConfig +
|
|
||||||
"/runs/" +
|
|
||||||
run +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// DataSourcePath returns the path for the data source resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/dataSources/%s", project, dataSource)
|
|
||||||
// instead.
|
|
||||||
func DataSourcePath(project, dataSource string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/dataSources/" +
|
|
||||||
dataSource +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// TransferConfigPath returns the path for the transfer config resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/transferConfigs/%s", project, transferConfig)
|
|
||||||
// instead.
|
|
||||||
func TransferConfigPath(project, transferConfig string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/transferConfigs/" +
|
|
||||||
transferConfig +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunPath returns the path for the run resource.
|
|
||||||
//
|
|
||||||
// Deprecated: Use
|
|
||||||
// fmt.Sprintf("projects/%s/transferConfigs/%s/runs/%s", project, transferConfig, run)
|
|
||||||
// instead.
|
|
||||||
func RunPath(project, transferConfig, run string) string {
|
|
||||||
return "" +
|
|
||||||
"projects/" +
|
|
||||||
project +
|
|
||||||
"/transferConfigs/" +
|
|
||||||
transferConfig +
|
|
||||||
"/runs/" +
|
|
||||||
run +
|
|
||||||
""
|
|
||||||
}
|
|
||||||
301
vendor/cloud.google.com/go/bigquery/doc.go
generated
vendored
301
vendor/cloud.google.com/go/bigquery/doc.go
generated
vendored
@ -1,301 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
/*
|
|
||||||
Package bigquery provides a client for the BigQuery service.
|
|
||||||
|
|
||||||
Note: This package is in beta. Some backwards-incompatible changes may occur.
|
|
||||||
|
|
||||||
The following assumes a basic familiarity with BigQuery concepts.
|
|
||||||
See https://cloud.google.com/bigquery/docs.
|
|
||||||
|
|
||||||
See https://godoc.org/cloud.google.com/go for authentication, timeouts,
|
|
||||||
connection pooling and similar aspects of this package.
|
|
||||||
|
|
||||||
|
|
||||||
Creating a Client
|
|
||||||
|
|
||||||
To start working with this package, create a client:
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, projectID)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Querying
|
|
||||||
|
|
||||||
To query existing tables, create a Query and call its Read method:
|
|
||||||
|
|
||||||
q := client.Query(`
|
|
||||||
SELECT year, SUM(number) as num
|
|
||||||
FROM [bigquery-public-data:usa_names.usa_1910_2013]
|
|
||||||
WHERE name = "William"
|
|
||||||
GROUP BY year
|
|
||||||
ORDER BY year
|
|
||||||
`)
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Then iterate through the resulting rows. You can store a row using
|
|
||||||
anything that implements the ValueLoader interface, or with a slice or map of bigquery.Value.
|
|
||||||
A slice is simplest:
|
|
||||||
|
|
||||||
for {
|
|
||||||
var values []bigquery.Value
|
|
||||||
err := it.Next(&values)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(values)
|
|
||||||
}
|
|
||||||
|
|
||||||
You can also use a struct whose exported fields match the query:
|
|
||||||
|
|
||||||
type Count struct {
|
|
||||||
Year int
|
|
||||||
Num int
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
var c Count
|
|
||||||
err := it.Next(&c)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
You can also start the query running and get the results later.
|
|
||||||
Create the query as above, but call Run instead of Read. This returns a Job,
|
|
||||||
which represents an asynchronous operation.
|
|
||||||
|
|
||||||
job, err := q.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Get the job's ID, a printable string. You can save this string to retrieve
|
|
||||||
the results at a later time, even in another process.
|
|
||||||
|
|
||||||
jobID := job.ID()
|
|
||||||
fmt.Printf("The job ID is %s\n", jobID)
|
|
||||||
|
|
||||||
To retrieve the job's results from the ID, first look up the Job:
|
|
||||||
|
|
||||||
job, err = client.JobFromID(ctx, jobID)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Use the Job.Read method to obtain an iterator, and loop over the rows.
|
|
||||||
Query.Read is just a convenience method that combines Query.Run and Job.Read.
|
|
||||||
|
|
||||||
it, err = job.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// Proceed with iteration as above.
|
|
||||||
|
|
||||||
Datasets and Tables
|
|
||||||
|
|
||||||
You can refer to datasets in the client's project with the Dataset method, and
|
|
||||||
in other projects with the DatasetInProject method:
|
|
||||||
|
|
||||||
myDataset := client.Dataset("my_dataset")
|
|
||||||
yourDataset := client.DatasetInProject("your-project-id", "your_dataset")
|
|
||||||
|
|
||||||
These methods create references to datasets, not the datasets themselves. You can have
|
|
||||||
a dataset reference even if the dataset doesn't exist yet. Use Dataset.Create to
|
|
||||||
create a dataset from a reference:
|
|
||||||
|
|
||||||
if err := myDataset.Create(ctx, nil); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
You can refer to tables with Dataset.Table. Like bigquery.Dataset, bigquery.Table is a reference
|
|
||||||
to an object in BigQuery that may or may not exist.
|
|
||||||
|
|
||||||
table := myDataset.Table("my_table")
|
|
||||||
|
|
||||||
You can create, delete and update the metadata of tables with methods on Table.
|
|
||||||
For instance, you could create a temporary table with:
|
|
||||||
|
|
||||||
err = myDataset.Table("temp").Create(ctx, &bigquery.TableMetadata{
|
|
||||||
ExpirationTime: time.Now().Add(1*time.Hour)})
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
We'll see how to create a table with a schema in the next section.
|
|
||||||
|
|
||||||
Schemas
|
|
||||||
|
|
||||||
There are two ways to construct schemas with this package.
|
|
||||||
You can build a schema by hand, like so:
|
|
||||||
|
|
||||||
schema1 := bigquery.Schema{
|
|
||||||
{Name: "Name", Required: true, Type: bigquery.StringFieldType},
|
|
||||||
{Name: "Grades", Repeated: true, Type: bigquery.IntegerFieldType},
|
|
||||||
{Name: "Optional", Required: false, Type: bigquery.IntegerFieldType},
|
|
||||||
}
|
|
||||||
|
|
||||||
Or you can infer the schema from a struct:
|
|
||||||
|
|
||||||
type student struct {
|
|
||||||
Name string
|
|
||||||
Grades []int
|
|
||||||
Optional bigquery.NullInt64
|
|
||||||
}
|
|
||||||
schema2, err := bigquery.InferSchema(student{})
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// schema1 and schema2 are identical.
|
|
||||||
|
|
||||||
Struct inference supports tags like those of the encoding/json package, so you can
|
|
||||||
change names, ignore fields, or mark a field as nullable (non-required). Fields
|
|
||||||
declared as one of the Null types (NullInt64, NullFloat64, NullString, NullBool,
|
|
||||||
NullTimestamp, NullDate, NullTime and NullDateTime) are automatically inferred as
|
|
||||||
nullable, so the "nullable" tag is only needed for []byte, *big.Rat and
|
|
||||||
pointer-to-struct fields.
|
|
||||||
|
|
||||||
type student2 struct {
|
|
||||||
Name string `bigquery:"full_name"`
|
|
||||||
Grades []int
|
|
||||||
Secret string `bigquery:"-"`
|
|
||||||
Optional []byte `bigquery:",nullable"
|
|
||||||
}
|
|
||||||
schema3, err := bigquery.InferSchema(student2{})
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// schema3 has required fields "full_name" and "Grade", and nullable BYTES field "Optional".
|
|
||||||
|
|
||||||
Having constructed a schema, you can create a table with it like so:
|
|
||||||
|
|
||||||
if err := table.Create(ctx, &bigquery.TableMetadata{Schema: schema1}); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Copying
|
|
||||||
|
|
||||||
You can copy one or more tables to another table. Begin by constructing a Copier
|
|
||||||
describing the copy. Then set any desired copy options, and finally call Run to get a Job:
|
|
||||||
|
|
||||||
copier := myDataset.Table("dest").CopierFrom(myDataset.Table("src"))
|
|
||||||
copier.WriteDisposition = bigquery.WriteTruncate
|
|
||||||
job, err = copier.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
You can chain the call to Run if you don't want to set options:
|
|
||||||
|
|
||||||
job, err = myDataset.Table("dest").CopierFrom(myDataset.Table("src")).Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
You can wait for your job to complete:
|
|
||||||
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Job.Wait polls with exponential backoff. You can also poll yourself, if you
|
|
||||||
wish:
|
|
||||||
|
|
||||||
for {
|
|
||||||
status, err := job.Status(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Done() {
|
|
||||||
if status.Err() != nil {
|
|
||||||
log.Fatalf("Job failed with error %v", status.Err())
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
Loading and Uploading
|
|
||||||
|
|
||||||
There are two ways to populate a table with this package: load the data from a Google Cloud Storage
|
|
||||||
object, or upload rows directly from your program.
|
|
||||||
|
|
||||||
For loading, first create a GCSReference, configuring it if desired. Then make a Loader, optionally configure
|
|
||||||
it as well, and call its Run method.
|
|
||||||
|
|
||||||
gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
|
|
||||||
gcsRef.AllowJaggedRows = true
|
|
||||||
loader := myDataset.Table("dest").LoaderFrom(gcsRef)
|
|
||||||
loader.CreateDisposition = bigquery.CreateNever
|
|
||||||
job, err = loader.Run(ctx)
|
|
||||||
// Poll the job for completion if desired, as above.
|
|
||||||
|
|
||||||
To upload, first define a type that implements the ValueSaver interface, which has a single method named Save.
|
|
||||||
Then create an Uploader, and call its Put method with a slice of values.
|
|
||||||
|
|
||||||
u := table.Uploader()
|
|
||||||
// Item implements the ValueSaver interface.
|
|
||||||
items := []*Item{
|
|
||||||
{Name: "n1", Size: 32.6, Count: 7},
|
|
||||||
{Name: "n2", Size: 4, Count: 2},
|
|
||||||
{Name: "n3", Size: 101.5, Count: 1},
|
|
||||||
}
|
|
||||||
if err := u.Put(ctx, items); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
You can also upload a struct that doesn't implement ValueSaver. Use the StructSaver type
|
|
||||||
to specify the schema and insert ID by hand, or just supply the struct or struct pointer
|
|
||||||
directly and the schema will be inferred:
|
|
||||||
|
|
||||||
type Item2 struct {
|
|
||||||
Name string
|
|
||||||
Size float64
|
|
||||||
Count int
|
|
||||||
}
|
|
||||||
// Item implements the ValueSaver interface.
|
|
||||||
items2 := []*Item2{
|
|
||||||
{Name: "n1", Size: 32.6, Count: 7},
|
|
||||||
{Name: "n2", Size: 4, Count: 2},
|
|
||||||
{Name: "n3", Size: 101.5, Count: 1},
|
|
||||||
}
|
|
||||||
if err := u.Put(ctx, items2); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
Extracting
|
|
||||||
|
|
||||||
If you've been following so far, extracting data from a BigQuery table
|
|
||||||
into a Google Cloud Storage object will feel familiar. First create an
|
|
||||||
Extractor, then optionally configure it, and lastly call its Run method.
|
|
||||||
|
|
||||||
extractor := table.ExtractorTo(gcsRef)
|
|
||||||
extractor.DisableHeader = true
|
|
||||||
job, err = extractor.Run(ctx)
|
|
||||||
// Poll the job for completion if desired, as above.
|
|
||||||
*/
|
|
||||||
package bigquery // import "cloud.google.com/go/bigquery"
|
|
||||||
82
vendor/cloud.google.com/go/bigquery/error.go
generated
vendored
82
vendor/cloud.google.com/go/bigquery/error.go
generated
vendored
@ -1,82 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// An Error contains detailed information about a failed bigquery operation.
|
|
||||||
type Error struct {
|
|
||||||
// Mirrors bq.ErrorProto, but drops DebugInfo
|
|
||||||
Location, Message, Reason string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e Error) Error() string {
|
|
||||||
return fmt.Sprintf("{Location: %q; Message: %q; Reason: %q}", e.Location, e.Message, e.Reason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToError(ep *bq.ErrorProto) *Error {
|
|
||||||
if ep == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &Error{
|
|
||||||
Location: ep.Location,
|
|
||||||
Message: ep.Message,
|
|
||||||
Reason: ep.Reason,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A MultiError contains multiple related errors.
|
|
||||||
type MultiError []error
|
|
||||||
|
|
||||||
func (m MultiError) Error() string {
|
|
||||||
switch len(m) {
|
|
||||||
case 0:
|
|
||||||
return "(0 errors)"
|
|
||||||
case 1:
|
|
||||||
return m[0].Error()
|
|
||||||
case 2:
|
|
||||||
return m[0].Error() + " (and 1 other error)"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s (and %d other errors)", m[0].Error(), len(m)-1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RowInsertionError contains all errors that occurred when attempting to insert a row.
|
|
||||||
type RowInsertionError struct {
|
|
||||||
InsertID string // The InsertID associated with the affected row.
|
|
||||||
RowIndex int // The 0-based index of the affected row in the batch of rows being inserted.
|
|
||||||
Errors MultiError
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *RowInsertionError) Error() string {
|
|
||||||
errFmt := "insertion of row [insertID: %q; insertIndex: %v] failed with error: %s"
|
|
||||||
return fmt.Sprintf(errFmt, e.InsertID, e.RowIndex, e.Errors.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutMultiError contains an error for each row which was not successfully inserted
|
|
||||||
// into a BigQuery table.
|
|
||||||
type PutMultiError []RowInsertionError
|
|
||||||
|
|
||||||
func (pme PutMultiError) Error() string {
|
|
||||||
plural := "s"
|
|
||||||
if len(pme) == 1 {
|
|
||||||
plural = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%v row insertion%s failed", len(pme), plural)
|
|
||||||
}
|
|
||||||
110
vendor/cloud.google.com/go/bigquery/error_test.go
generated
vendored
110
vendor/cloud.google.com/go/bigquery/error_test.go
generated
vendored
@ -1,110 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func rowInsertionError(msg string) RowInsertionError {
|
|
||||||
return RowInsertionError{Errors: []error{errors.New(msg)}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPutMultiErrorString(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
errs PutMultiError
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
errs: PutMultiError{},
|
|
||||||
want: "0 row insertions failed",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
errs: PutMultiError{rowInsertionError("a")},
|
|
||||||
want: "1 row insertion failed",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
errs: PutMultiError{rowInsertionError("a"), rowInsertionError("b")},
|
|
||||||
want: "2 row insertions failed",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
if tc.errs.Error() != tc.want {
|
|
||||||
t.Errorf("PutMultiError string: got:\n%v\nwant:\n%v", tc.errs.Error(), tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultiErrorString(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
errs MultiError
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
errs: MultiError{},
|
|
||||||
want: "(0 errors)",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
errs: MultiError{errors.New("a")},
|
|
||||||
want: "a",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
errs: MultiError{errors.New("a"), errors.New("b")},
|
|
||||||
want: "a (and 1 other error)",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
errs: MultiError{errors.New("a"), errors.New("b"), errors.New("c")},
|
|
||||||
want: "a (and 2 other errors)",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
if tc.errs.Error() != tc.want {
|
|
||||||
t.Errorf("PutMultiError string: got:\n%v\nwant:\n%v", tc.errs.Error(), tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrorFromErrorProto(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
in *bq.ErrorProto
|
|
||||||
want *Error
|
|
||||||
}{
|
|
||||||
{nil, nil},
|
|
||||||
{
|
|
||||||
in: &bq.ErrorProto{Location: "L", Message: "M", Reason: "R"},
|
|
||||||
want: &Error{Location: "L", Message: "M", Reason: "R"},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
if got := bqToError(test.in); !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%v: got %v, want %v", test.in, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrorString(t *testing.T) {
|
|
||||||
e := &Error{Location: "<L>", Message: "<M>", Reason: "<R>"}
|
|
||||||
got := e.Error()
|
|
||||||
if !strings.Contains(got, "<L>") || !strings.Contains(got, "<M>") || !strings.Contains(got, "<R>") {
|
|
||||||
t.Errorf(`got %q, expected to see "<L>", "<M>" and "<R>"`, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
829
vendor/cloud.google.com/go/bigquery/examples_test.go
generated
vendored
829
vendor/cloud.google.com/go/bigquery/examples_test.go
generated
vendored
@ -1,829 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigquery"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ExampleNewClient() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
_ = client // TODO: Use client.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_Dataset() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
fmt.Println(ds)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_DatasetInProject() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.DatasetInProject("their-project-id", "their-dataset")
|
|
||||||
fmt.Println(ds)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_Datasets() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Datasets(ctx)
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_DatasetsInProject() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.DatasetsInProject(ctx, "their-project-id")
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func getJobID() string { return "" }
|
|
||||||
|
|
||||||
func ExampleClient_JobFromID() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
jobID := getJobID() // Get a job ID using Job.ID, the console or elsewhere.
|
|
||||||
job, err := client.JobFromID(ctx, jobID)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(job.LastStatus()) // Display the job's status.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_Jobs() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Jobs(ctx)
|
|
||||||
it.State = bigquery.Running // list only running jobs.
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleNewGCSReference() {
|
|
||||||
gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
|
|
||||||
fmt.Println(gcsRef)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_Query() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
q.DefaultProjectID = "project-id"
|
|
||||||
// TODO: set other options on the Query.
|
|
||||||
// TODO: Call Query.Run or Query.Read.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleClient_Query_parameters() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select num from t1 where name = @user")
|
|
||||||
q.Parameters = []bigquery.QueryParameter{
|
|
||||||
{Name: "user", Value: "Elizabeth"},
|
|
||||||
}
|
|
||||||
// TODO: set other options on the Query.
|
|
||||||
// TODO: Call Query.Run or Query.Read.
|
|
||||||
}
|
|
||||||
|
|
||||||
// This example demonstrates how to run a query job on a table
|
|
||||||
// with a customer-managed encryption key. The same
|
|
||||||
// applies to load and copy jobs as well.
|
|
||||||
func ExampleClient_Query_encryptionKey() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
// TODO: Replace this key with a key you have created in Cloud KMS.
|
|
||||||
keyName := "projects/P/locations/L/keyRings/R/cryptoKeys/K"
|
|
||||||
q.DestinationEncryptionConfig = &bigquery.EncryptionConfig{KMSKeyName: keyName}
|
|
||||||
// TODO: set other options on the Query.
|
|
||||||
// TODO: Call Query.Run or Query.Read.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleQuery_Read() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleRowIterator_Next() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
var row []bigquery.Value
|
|
||||||
err := it.Next(&row)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(row)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleRowIterator_Next_struct() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
type score struct {
|
|
||||||
Name string
|
|
||||||
Num int
|
|
||||||
}
|
|
||||||
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
it, err := q.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
var s score
|
|
||||||
err := it.Next(&s)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleJob_Read() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
q := client.Query("select name, num from t1")
|
|
||||||
// Call Query.Run to get a Job, then call Read on the job.
|
|
||||||
// Note: Query.Read is a shorthand for this.
|
|
||||||
job, err := q.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it, err := job.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleJob_Wait() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
job, err := ds.Table("t1").CopierFrom(ds.Table("t2")).Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Err() != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleJob_Config() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
job, err := ds.Table("t1").CopierFrom(ds.Table("t2")).Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
jc, err := job.Config()
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
copyConfig := jc.(*bigquery.CopyConfig)
|
|
||||||
fmt.Println(copyConfig.Dst, copyConfig.CreateDisposition)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDataset_Create() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
if err := ds.Create(ctx, &bigquery.DatasetMetadata{Location: "EU"}); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDataset_Delete() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if err := client.Dataset("my_dataset").Delete(ctx); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDataset_Metadata() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
md, err := client.Dataset("my_dataset").Metadata(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(md)
|
|
||||||
}
|
|
||||||
|
|
||||||
// This example illustrates how to perform a read-modify-write sequence on dataset
|
|
||||||
// metadata. Passing the metadata's ETag to the Update call ensures that the call
|
|
||||||
// will fail if the metadata was changed since the read.
|
|
||||||
func ExampleDataset_Update_readModifyWrite() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
md, err := ds.Metadata(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
md2, err := ds.Update(ctx,
|
|
||||||
bigquery.DatasetMetadataToUpdate{Name: "new " + md.Name},
|
|
||||||
md.ETag)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(md2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// To perform a blind write, ignoring the existing state (and possibly overwriting
|
|
||||||
// other updates), pass the empty string as the etag.
|
|
||||||
func ExampleDataset_Update_blindWrite() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
md, err := client.Dataset("my_dataset").Update(ctx, bigquery.DatasetMetadataToUpdate{Name: "blind"}, "")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(md)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDataset_Table() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
// Table creates a reference to the table. It does not create the actual
|
|
||||||
// table in BigQuery; to do so, use Table.Create.
|
|
||||||
t := client.Dataset("my_dataset").Table("my_table")
|
|
||||||
fmt.Println(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDataset_Tables() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Dataset("my_dataset").Tables(ctx)
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleDatasetIterator_Next() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Datasets(ctx)
|
|
||||||
for {
|
|
||||||
ds, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(ds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleInferSchema() {
|
|
||||||
type Item struct {
|
|
||||||
Name string
|
|
||||||
Size float64
|
|
||||||
Count int
|
|
||||||
}
|
|
||||||
schema, err := bigquery.InferSchema(Item{})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println(err)
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
for _, fs := range schema {
|
|
||||||
fmt.Println(fs.Name, fs.Type)
|
|
||||||
}
|
|
||||||
// Output:
|
|
||||||
// Name STRING
|
|
||||||
// Size FLOAT
|
|
||||||
// Count INTEGER
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleInferSchema_tags() {
|
|
||||||
type Item struct {
|
|
||||||
Name string
|
|
||||||
Size float64
|
|
||||||
Count int `bigquery:"number"`
|
|
||||||
Secret []byte `bigquery:"-"`
|
|
||||||
Optional bigquery.NullBool
|
|
||||||
OptBytes []byte `bigquery:",nullable"`
|
|
||||||
}
|
|
||||||
schema, err := bigquery.InferSchema(Item{})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println(err)
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
for _, fs := range schema {
|
|
||||||
fmt.Println(fs.Name, fs.Type, fs.Required)
|
|
||||||
}
|
|
||||||
// Output:
|
|
||||||
// Name STRING true
|
|
||||||
// Size FLOAT true
|
|
||||||
// number INTEGER true
|
|
||||||
// Optional BOOLEAN false
|
|
||||||
// OptBytes BYTES false
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Create() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
t := client.Dataset("my_dataset").Table("new-table")
|
|
||||||
if err := t.Create(ctx, nil); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize a new table by passing TableMetadata to Table.Create.
|
|
||||||
func ExampleTable_Create_initialize() {
|
|
||||||
ctx := context.Background()
|
|
||||||
// Infer table schema from a Go type.
|
|
||||||
schema, err := bigquery.InferSchema(Item{})
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
t := client.Dataset("my_dataset").Table("new-table")
|
|
||||||
if err := t.Create(ctx,
|
|
||||||
&bigquery.TableMetadata{
|
|
||||||
Name: "My New Table",
|
|
||||||
Schema: schema,
|
|
||||||
ExpirationTime: time.Now().Add(24 * time.Hour),
|
|
||||||
}); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// This example demonstrates how to create a table with
|
|
||||||
// a customer-managed encryption key.
|
|
||||||
func ExampleTable_Create_encryptionKey() {
|
|
||||||
ctx := context.Background()
|
|
||||||
// Infer table schema from a Go type.
|
|
||||||
schema, err := bigquery.InferSchema(Item{})
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
t := client.Dataset("my_dataset").Table("new-table")
|
|
||||||
|
|
||||||
// TODO: Replace this key with a key you have created in Cloud KMS.
|
|
||||||
keyName := "projects/P/locations/L/keyRings/R/cryptoKeys/K"
|
|
||||||
if err := t.Create(ctx,
|
|
||||||
&bigquery.TableMetadata{
|
|
||||||
Name: "My New Table",
|
|
||||||
Schema: schema,
|
|
||||||
EncryptionConfig: &bigquery.EncryptionConfig{KMSKeyName: keyName},
|
|
||||||
}); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Delete() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if err := client.Dataset("my_dataset").Table("my_table").Delete(ctx); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Metadata() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
md, err := client.Dataset("my_dataset").Table("my_table").Metadata(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(md)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Uploader() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
_ = u // TODO: Use u.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Uploader_options() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
u.SkipInvalidRows = true
|
|
||||||
u.IgnoreUnknownValues = true
|
|
||||||
_ = u // TODO: Use u.
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_CopierFrom() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
c := ds.Table("combined").CopierFrom(ds.Table("t1"), ds.Table("t2"))
|
|
||||||
c.WriteDisposition = bigquery.WriteTruncate
|
|
||||||
// TODO: set other options on the Copier.
|
|
||||||
job, err := c.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Err() != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_ExtractorTo() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
|
|
||||||
gcsRef.FieldDelimiter = ":"
|
|
||||||
// TODO: set other options on the GCSReference.
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
extractor := ds.Table("my_table").ExtractorTo(gcsRef)
|
|
||||||
extractor.DisableHeader = true
|
|
||||||
// TODO: set other options on the Extractor.
|
|
||||||
job, err := extractor.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Err() != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_LoaderFrom() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
|
|
||||||
gcsRef.AllowJaggedRows = true
|
|
||||||
gcsRef.MaxBadRecords = 5
|
|
||||||
gcsRef.Schema = schema
|
|
||||||
// TODO: set other options on the GCSReference.
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
loader := ds.Table("my_table").LoaderFrom(gcsRef)
|
|
||||||
loader.CreateDisposition = bigquery.CreateNever
|
|
||||||
// TODO: set other options on the Loader.
|
|
||||||
job, err := loader.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Err() != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_LoaderFrom_reader() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
f, err := os.Open("data.csv")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
rs := bigquery.NewReaderSource(f)
|
|
||||||
rs.AllowJaggedRows = true
|
|
||||||
rs.MaxBadRecords = 5
|
|
||||||
rs.Schema = schema
|
|
||||||
// TODO: set other options on the GCSReference.
|
|
||||||
ds := client.Dataset("my_dataset")
|
|
||||||
loader := ds.Table("my_table").LoaderFrom(rs)
|
|
||||||
loader.CreateDisposition = bigquery.CreateNever
|
|
||||||
// TODO: set other options on the Loader.
|
|
||||||
job, err := loader.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
status, err := job.Wait(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
if status.Err() != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTable_Read() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Dataset("my_dataset").Table("my_table").Read(ctx)
|
|
||||||
_ = it // TODO: iterate using Next or iterator.Pager.
|
|
||||||
}
|
|
||||||
|
|
||||||
// This example illustrates how to perform a read-modify-write sequence on table
|
|
||||||
// metadata. Passing the metadata's ETag to the Update call ensures that the call
|
|
||||||
// will fail if the metadata was changed since the read.
|
|
||||||
func ExampleTable_Update_readModifyWrite() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
t := client.Dataset("my_dataset").Table("my_table")
|
|
||||||
md, err := t.Metadata(ctx)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
md2, err := t.Update(ctx,
|
|
||||||
bigquery.TableMetadataToUpdate{Name: "new " + md.Name},
|
|
||||||
md.ETag)
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(md2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// To perform a blind write, ignoring the existing state (and possibly overwriting
|
|
||||||
// other updates), pass the empty string as the etag.
|
|
||||||
func ExampleTable_Update_blindWrite() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
t := client.Dataset("my_dataset").Table("my_table")
|
|
||||||
tm, err := t.Update(ctx, bigquery.TableMetadataToUpdate{
|
|
||||||
Description: "my favorite table",
|
|
||||||
}, "")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(tm)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleTableIterator_Next() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
it := client.Dataset("my_dataset").Tables(ctx)
|
|
||||||
for {
|
|
||||||
t, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
fmt.Println(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type Item struct {
|
|
||||||
Name string
|
|
||||||
Size float64
|
|
||||||
Count int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save implements the ValueSaver interface.
|
|
||||||
func (i *Item) Save() (map[string]bigquery.Value, string, error) {
|
|
||||||
return map[string]bigquery.Value{
|
|
||||||
"Name": i.Name,
|
|
||||||
"Size": i.Size,
|
|
||||||
"Count": i.Count,
|
|
||||||
}, "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleUploader_Put() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
// Item implements the ValueSaver interface.
|
|
||||||
items := []*Item{
|
|
||||||
{Name: "n1", Size: 32.6, Count: 7},
|
|
||||||
{Name: "n2", Size: 4, Count: 2},
|
|
||||||
{Name: "n3", Size: 101.5, Count: 1},
|
|
||||||
}
|
|
||||||
if err := u.Put(ctx, items); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var schema bigquery.Schema
|
|
||||||
|
|
||||||
func ExampleUploader_Put_structSaver() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
|
|
||||||
type score struct {
|
|
||||||
Name string
|
|
||||||
Num int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assume schema holds the table's schema.
|
|
||||||
savers := []*bigquery.StructSaver{
|
|
||||||
{Struct: score{Name: "n1", Num: 12}, Schema: schema, InsertID: "id1"},
|
|
||||||
{Struct: score{Name: "n2", Num: 31}, Schema: schema, InsertID: "id2"},
|
|
||||||
{Struct: score{Name: "n3", Num: 7}, Schema: schema, InsertID: "id3"},
|
|
||||||
}
|
|
||||||
if err := u.Put(ctx, savers); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleUploader_Put_struct() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
|
|
||||||
type score struct {
|
|
||||||
Name string
|
|
||||||
Num int
|
|
||||||
}
|
|
||||||
scores := []score{
|
|
||||||
{Name: "n1", Num: 12},
|
|
||||||
{Name: "n2", Num: 31},
|
|
||||||
{Name: "n3", Num: 7},
|
|
||||||
}
|
|
||||||
// Schema is inferred from the score type.
|
|
||||||
if err := u.Put(ctx, scores); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ExampleUploader_Put_valuesSaver() {
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := bigquery.NewClient(ctx, "project-id")
|
|
||||||
if err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
|
|
||||||
u := client.Dataset("my_dataset").Table("my_table").Uploader()
|
|
||||||
|
|
||||||
var vss []*bigquery.ValuesSaver
|
|
||||||
for i, name := range []string{"n1", "n2", "n3"} {
|
|
||||||
// Assume schema holds the table's schema.
|
|
||||||
vss = append(vss, &bigquery.ValuesSaver{
|
|
||||||
Schema: schema,
|
|
||||||
InsertID: name,
|
|
||||||
Row: []bigquery.Value{name, int64(i)},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := u.Put(ctx, vss); err != nil {
|
|
||||||
// TODO: Handle error.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
399
vendor/cloud.google.com/go/bigquery/external.go
generated
vendored
399
vendor/cloud.google.com/go/bigquery/external.go
generated
vendored
@ -1,399 +0,0 @@
|
|||||||
// Copyright 2017 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DataFormat describes the format of BigQuery table data.
|
|
||||||
type DataFormat string
|
|
||||||
|
|
||||||
// Constants describing the format of BigQuery table data.
|
|
||||||
const (
|
|
||||||
CSV DataFormat = "CSV"
|
|
||||||
Avro DataFormat = "AVRO"
|
|
||||||
JSON DataFormat = "NEWLINE_DELIMITED_JSON"
|
|
||||||
DatastoreBackup DataFormat = "DATASTORE_BACKUP"
|
|
||||||
GoogleSheets DataFormat = "GOOGLE_SHEETS"
|
|
||||||
Bigtable DataFormat = "BIGTABLE"
|
|
||||||
Parquet DataFormat = "PARQUET"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ExternalData is a table which is stored outside of BigQuery. It is implemented by
|
|
||||||
// *ExternalDataConfig.
|
|
||||||
// GCSReference also implements it, for backwards compatibility.
|
|
||||||
type ExternalData interface {
|
|
||||||
toBQ() bq.ExternalDataConfiguration
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExternalDataConfig describes data external to BigQuery that can be used
|
|
||||||
// in queries and to create external tables.
|
|
||||||
type ExternalDataConfig struct {
|
|
||||||
// The format of the data. Required.
|
|
||||||
SourceFormat DataFormat
|
|
||||||
|
|
||||||
// The fully-qualified URIs that point to your
|
|
||||||
// data in Google Cloud. Required.
|
|
||||||
//
|
|
||||||
// For Google Cloud Storage URIs, each URI can contain one '*' wildcard character
|
|
||||||
// and it must come after the 'bucket' name. Size limits related to load jobs
|
|
||||||
// apply to external data sources.
|
|
||||||
//
|
|
||||||
// For Google Cloud Bigtable URIs, exactly one URI can be specified and it has be
|
|
||||||
// a fully specified and valid HTTPS URL for a Google Cloud Bigtable table.
|
|
||||||
//
|
|
||||||
// For Google Cloud Datastore backups, exactly one URI can be specified. Also,
|
|
||||||
// the '*' wildcard character is not allowed.
|
|
||||||
SourceURIs []string
|
|
||||||
|
|
||||||
// The schema of the data. Required for CSV and JSON; disallowed for the
|
|
||||||
// other formats.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// Try to detect schema and format options automatically.
|
|
||||||
// Any option specified explicitly will be honored.
|
|
||||||
AutoDetect bool
|
|
||||||
|
|
||||||
// The compression type of the data.
|
|
||||||
Compression Compression
|
|
||||||
|
|
||||||
// IgnoreUnknownValues causes values not matching the schema to be
|
|
||||||
// tolerated. Unknown values are ignored. For CSV this ignores extra values
|
|
||||||
// at the end of a line. For JSON this ignores named values that do not
|
|
||||||
// match any column name. If this field is not set, records containing
|
|
||||||
// unknown values are treated as bad records. The MaxBadRecords field can
|
|
||||||
// be used to customize how bad records are handled.
|
|
||||||
IgnoreUnknownValues bool
|
|
||||||
|
|
||||||
// MaxBadRecords is the maximum number of bad records that will be ignored
|
|
||||||
// when reading data.
|
|
||||||
MaxBadRecords int64
|
|
||||||
|
|
||||||
// Additional options for CSV, GoogleSheets and Bigtable formats.
|
|
||||||
Options ExternalDataConfigOptions
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *ExternalDataConfig) toBQ() bq.ExternalDataConfiguration {
|
|
||||||
q := bq.ExternalDataConfiguration{
|
|
||||||
SourceFormat: string(e.SourceFormat),
|
|
||||||
SourceUris: e.SourceURIs,
|
|
||||||
Autodetect: e.AutoDetect,
|
|
||||||
Compression: string(e.Compression),
|
|
||||||
IgnoreUnknownValues: e.IgnoreUnknownValues,
|
|
||||||
MaxBadRecords: e.MaxBadRecords,
|
|
||||||
}
|
|
||||||
if e.Schema != nil {
|
|
||||||
q.Schema = e.Schema.toBQ()
|
|
||||||
}
|
|
||||||
if e.Options != nil {
|
|
||||||
e.Options.populateExternalDataConfig(&q)
|
|
||||||
}
|
|
||||||
return q
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToExternalDataConfig(q *bq.ExternalDataConfiguration) (*ExternalDataConfig, error) {
|
|
||||||
e := &ExternalDataConfig{
|
|
||||||
SourceFormat: DataFormat(q.SourceFormat),
|
|
||||||
SourceURIs: q.SourceUris,
|
|
||||||
AutoDetect: q.Autodetect,
|
|
||||||
Compression: Compression(q.Compression),
|
|
||||||
IgnoreUnknownValues: q.IgnoreUnknownValues,
|
|
||||||
MaxBadRecords: q.MaxBadRecords,
|
|
||||||
Schema: bqToSchema(q.Schema),
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case q.CsvOptions != nil:
|
|
||||||
e.Options = bqToCSVOptions(q.CsvOptions)
|
|
||||||
case q.GoogleSheetsOptions != nil:
|
|
||||||
e.Options = bqToGoogleSheetsOptions(q.GoogleSheetsOptions)
|
|
||||||
case q.BigtableOptions != nil:
|
|
||||||
var err error
|
|
||||||
e.Options, err = bqToBigtableOptions(q.BigtableOptions)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return e, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExternalDataConfigOptions are additional options for external data configurations.
|
|
||||||
// This interface is implemented by CSVOptions, GoogleSheetsOptions and BigtableOptions.
|
|
||||||
type ExternalDataConfigOptions interface {
|
|
||||||
populateExternalDataConfig(*bq.ExternalDataConfiguration)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CSVOptions are additional options for CSV external data sources.
|
|
||||||
type CSVOptions struct {
|
|
||||||
// AllowJaggedRows causes missing trailing optional columns to be tolerated
|
|
||||||
// when reading CSV data. Missing values are treated as nulls.
|
|
||||||
AllowJaggedRows bool
|
|
||||||
|
|
||||||
// AllowQuotedNewlines sets whether quoted data sections containing
|
|
||||||
// newlines are allowed when reading CSV data.
|
|
||||||
AllowQuotedNewlines bool
|
|
||||||
|
|
||||||
// Encoding is the character encoding of data to be read.
|
|
||||||
Encoding Encoding
|
|
||||||
|
|
||||||
// FieldDelimiter is the separator for fields in a CSV file, used when
|
|
||||||
// reading or exporting data. The default is ",".
|
|
||||||
FieldDelimiter string
|
|
||||||
|
|
||||||
// Quote is the value used to quote data sections in a CSV file. The
|
|
||||||
// default quotation character is the double quote ("), which is used if
|
|
||||||
// both Quote and ForceZeroQuote are unset.
|
|
||||||
// To specify that no character should be interpreted as a quotation
|
|
||||||
// character, set ForceZeroQuote to true.
|
|
||||||
// Only used when reading data.
|
|
||||||
Quote string
|
|
||||||
ForceZeroQuote bool
|
|
||||||
|
|
||||||
// The number of rows at the top of a CSV file that BigQuery will skip when
|
|
||||||
// reading data.
|
|
||||||
SkipLeadingRows int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *CSVOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
|
|
||||||
c.CsvOptions = &bq.CsvOptions{
|
|
||||||
AllowJaggedRows: o.AllowJaggedRows,
|
|
||||||
AllowQuotedNewlines: o.AllowQuotedNewlines,
|
|
||||||
Encoding: string(o.Encoding),
|
|
||||||
FieldDelimiter: o.FieldDelimiter,
|
|
||||||
Quote: o.quote(),
|
|
||||||
SkipLeadingRows: o.SkipLeadingRows,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// quote returns the CSV quote character, or nil if unset.
|
|
||||||
func (o *CSVOptions) quote() *string {
|
|
||||||
if o.ForceZeroQuote {
|
|
||||||
quote := ""
|
|
||||||
return "e
|
|
||||||
}
|
|
||||||
if o.Quote == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &o.Quote
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *CSVOptions) setQuote(ps *string) {
|
|
||||||
if ps != nil {
|
|
||||||
o.Quote = *ps
|
|
||||||
if o.Quote == "" {
|
|
||||||
o.ForceZeroQuote = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToCSVOptions(q *bq.CsvOptions) *CSVOptions {
|
|
||||||
o := &CSVOptions{
|
|
||||||
AllowJaggedRows: q.AllowJaggedRows,
|
|
||||||
AllowQuotedNewlines: q.AllowQuotedNewlines,
|
|
||||||
Encoding: Encoding(q.Encoding),
|
|
||||||
FieldDelimiter: q.FieldDelimiter,
|
|
||||||
SkipLeadingRows: q.SkipLeadingRows,
|
|
||||||
}
|
|
||||||
o.setQuote(q.Quote)
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// GoogleSheetsOptions are additional options for GoogleSheets external data sources.
|
|
||||||
type GoogleSheetsOptions struct {
|
|
||||||
// The number of rows at the top of a sheet that BigQuery will skip when
|
|
||||||
// reading data.
|
|
||||||
SkipLeadingRows int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *GoogleSheetsOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
|
|
||||||
c.GoogleSheetsOptions = &bq.GoogleSheetsOptions{
|
|
||||||
SkipLeadingRows: o.SkipLeadingRows,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToGoogleSheetsOptions(q *bq.GoogleSheetsOptions) *GoogleSheetsOptions {
|
|
||||||
return &GoogleSheetsOptions{
|
|
||||||
SkipLeadingRows: q.SkipLeadingRows,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// BigtableOptions are additional options for Bigtable external data sources.
|
|
||||||
type BigtableOptions struct {
|
|
||||||
// A list of column families to expose in the table schema along with their
|
|
||||||
// types. If omitted, all column families are present in the table schema and
|
|
||||||
// their values are read as BYTES.
|
|
||||||
ColumnFamilies []*BigtableColumnFamily
|
|
||||||
|
|
||||||
// If true, then the column families that are not specified in columnFamilies
|
|
||||||
// list are not exposed in the table schema. Otherwise, they are read with BYTES
|
|
||||||
// type values. The default is false.
|
|
||||||
IgnoreUnspecifiedColumnFamilies bool
|
|
||||||
|
|
||||||
// If true, then the rowkey column families will be read and converted to string.
|
|
||||||
// Otherwise they are read with BYTES type values and users need to manually cast
|
|
||||||
// them with CAST if necessary. The default is false.
|
|
||||||
ReadRowkeyAsString bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *BigtableOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
|
|
||||||
q := &bq.BigtableOptions{
|
|
||||||
IgnoreUnspecifiedColumnFamilies: o.IgnoreUnspecifiedColumnFamilies,
|
|
||||||
ReadRowkeyAsString: o.ReadRowkeyAsString,
|
|
||||||
}
|
|
||||||
for _, f := range o.ColumnFamilies {
|
|
||||||
q.ColumnFamilies = append(q.ColumnFamilies, f.toBQ())
|
|
||||||
}
|
|
||||||
c.BigtableOptions = q
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToBigtableOptions(q *bq.BigtableOptions) (*BigtableOptions, error) {
|
|
||||||
b := &BigtableOptions{
|
|
||||||
IgnoreUnspecifiedColumnFamilies: q.IgnoreUnspecifiedColumnFamilies,
|
|
||||||
ReadRowkeyAsString: q.ReadRowkeyAsString,
|
|
||||||
}
|
|
||||||
for _, f := range q.ColumnFamilies {
|
|
||||||
f2, err := bqToBigtableColumnFamily(f)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
b.ColumnFamilies = append(b.ColumnFamilies, f2)
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BigtableColumnFamily describes how BigQuery should access a Bigtable column family.
|
|
||||||
type BigtableColumnFamily struct {
|
|
||||||
// Identifier of the column family.
|
|
||||||
FamilyID string
|
|
||||||
|
|
||||||
// Lists of columns that should be exposed as individual fields as opposed to a
|
|
||||||
// list of (column name, value) pairs. All columns whose qualifier matches a
|
|
||||||
// qualifier in this list can be accessed as .. Other columns can be accessed as
|
|
||||||
// a list through .Column field.
|
|
||||||
Columns []*BigtableColumn
|
|
||||||
|
|
||||||
// The encoding of the values when the type is not STRING. Acceptable encoding values are:
|
|
||||||
// - TEXT - indicates values are alphanumeric text strings.
|
|
||||||
// - BINARY - indicates values are encoded using HBase Bytes.toBytes family of functions.
|
|
||||||
// This can be overridden for a specific column by listing that column in 'columns' and
|
|
||||||
// specifying an encoding for it.
|
|
||||||
Encoding string
|
|
||||||
|
|
||||||
// If true, only the latest version of values are exposed for all columns in this
|
|
||||||
// column family. This can be overridden for a specific column by listing that
|
|
||||||
// column in 'columns' and specifying a different setting for that column.
|
|
||||||
OnlyReadLatest bool
|
|
||||||
|
|
||||||
// The type to convert the value in cells of this
|
|
||||||
// column family. The values are expected to be encoded using HBase
|
|
||||||
// Bytes.toBytes function when using the BINARY encoding value.
|
|
||||||
// Following BigQuery types are allowed (case-sensitive):
|
|
||||||
// BYTES STRING INTEGER FLOAT BOOLEAN.
|
|
||||||
// The default type is BYTES. This can be overridden for a specific column by
|
|
||||||
// listing that column in 'columns' and specifying a type for it.
|
|
||||||
Type string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *BigtableColumnFamily) toBQ() *bq.BigtableColumnFamily {
|
|
||||||
q := &bq.BigtableColumnFamily{
|
|
||||||
FamilyId: b.FamilyID,
|
|
||||||
Encoding: b.Encoding,
|
|
||||||
OnlyReadLatest: b.OnlyReadLatest,
|
|
||||||
Type: b.Type,
|
|
||||||
}
|
|
||||||
for _, col := range b.Columns {
|
|
||||||
q.Columns = append(q.Columns, col.toBQ())
|
|
||||||
}
|
|
||||||
return q
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToBigtableColumnFamily(q *bq.BigtableColumnFamily) (*BigtableColumnFamily, error) {
|
|
||||||
b := &BigtableColumnFamily{
|
|
||||||
FamilyID: q.FamilyId,
|
|
||||||
Encoding: q.Encoding,
|
|
||||||
OnlyReadLatest: q.OnlyReadLatest,
|
|
||||||
Type: q.Type,
|
|
||||||
}
|
|
||||||
for _, col := range q.Columns {
|
|
||||||
c, err := bqToBigtableColumn(col)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
b.Columns = append(b.Columns, c)
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BigtableColumn describes how BigQuery should access a Bigtable column.
|
|
||||||
type BigtableColumn struct {
|
|
||||||
// Qualifier of the column. Columns in the parent column family that have this
|
|
||||||
// exact qualifier are exposed as . field. The column field name is the
|
|
||||||
// same as the column qualifier.
|
|
||||||
Qualifier string
|
|
||||||
|
|
||||||
// If the qualifier is not a valid BigQuery field identifier i.e. does not match
|
|
||||||
// [a-zA-Z][a-zA-Z0-9_]*, a valid identifier must be provided as the column field
|
|
||||||
// name and is used as field name in queries.
|
|
||||||
FieldName string
|
|
||||||
|
|
||||||
// If true, only the latest version of values are exposed for this column.
|
|
||||||
// See BigtableColumnFamily.OnlyReadLatest.
|
|
||||||
OnlyReadLatest bool
|
|
||||||
|
|
||||||
// The encoding of the values when the type is not STRING.
|
|
||||||
// See BigtableColumnFamily.Encoding
|
|
||||||
Encoding string
|
|
||||||
|
|
||||||
// The type to convert the value in cells of this column.
|
|
||||||
// See BigtableColumnFamily.Type
|
|
||||||
Type string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *BigtableColumn) toBQ() *bq.BigtableColumn {
|
|
||||||
q := &bq.BigtableColumn{
|
|
||||||
FieldName: b.FieldName,
|
|
||||||
OnlyReadLatest: b.OnlyReadLatest,
|
|
||||||
Encoding: b.Encoding,
|
|
||||||
Type: b.Type,
|
|
||||||
}
|
|
||||||
if utf8.ValidString(b.Qualifier) {
|
|
||||||
q.QualifierString = b.Qualifier
|
|
||||||
} else {
|
|
||||||
q.QualifierEncoded = base64.RawStdEncoding.EncodeToString([]byte(b.Qualifier))
|
|
||||||
}
|
|
||||||
return q
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToBigtableColumn(q *bq.BigtableColumn) (*BigtableColumn, error) {
|
|
||||||
b := &BigtableColumn{
|
|
||||||
FieldName: q.FieldName,
|
|
||||||
OnlyReadLatest: q.OnlyReadLatest,
|
|
||||||
Encoding: q.Encoding,
|
|
||||||
Type: q.Type,
|
|
||||||
}
|
|
||||||
if q.QualifierString != "" {
|
|
||||||
b.Qualifier = q.QualifierString
|
|
||||||
} else {
|
|
||||||
bytes, err := base64.RawStdEncoding.DecodeString(q.QualifierEncoded)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
b.Qualifier = string(bytes)
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
143
vendor/cloud.google.com/go/bigquery/external_test.go
generated
vendored
143
vendor/cloud.google.com/go/bigquery/external_test.go
generated
vendored
@ -1,143 +0,0 @@
|
|||||||
// Copyright 2017 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/pretty"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestExternalDataConfig(t *testing.T) {
|
|
||||||
// Round-trip of ExternalDataConfig to underlying representation.
|
|
||||||
for i, want := range []*ExternalDataConfig{
|
|
||||||
{
|
|
||||||
SourceFormat: CSV,
|
|
||||||
SourceURIs: []string{"uri"},
|
|
||||||
Schema: Schema{{Name: "n", Type: IntegerFieldType}},
|
|
||||||
AutoDetect: true,
|
|
||||||
Compression: Gzip,
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
MaxBadRecords: 17,
|
|
||||||
Options: &CSVOptions{
|
|
||||||
AllowJaggedRows: true,
|
|
||||||
AllowQuotedNewlines: true,
|
|
||||||
Encoding: UTF_8,
|
|
||||||
FieldDelimiter: "f",
|
|
||||||
Quote: "q",
|
|
||||||
SkipLeadingRows: 3,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SourceFormat: GoogleSheets,
|
|
||||||
Options: &GoogleSheetsOptions{SkipLeadingRows: 4},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SourceFormat: Bigtable,
|
|
||||||
Options: &BigtableOptions{
|
|
||||||
IgnoreUnspecifiedColumnFamilies: true,
|
|
||||||
ReadRowkeyAsString: true,
|
|
||||||
ColumnFamilies: []*BigtableColumnFamily{
|
|
||||||
{
|
|
||||||
FamilyID: "f1",
|
|
||||||
Encoding: "TEXT",
|
|
||||||
OnlyReadLatest: true,
|
|
||||||
Type: "FLOAT",
|
|
||||||
Columns: []*BigtableColumn{
|
|
||||||
{
|
|
||||||
Qualifier: "valid-utf-8",
|
|
||||||
FieldName: "fn",
|
|
||||||
OnlyReadLatest: true,
|
|
||||||
Encoding: "BINARY",
|
|
||||||
Type: "STRING",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
q := want.toBQ()
|
|
||||||
got, err := bqToExternalDataConfig(&q)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Errorf("#%d: got=-, want=+:\n%s", i, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuote(t *testing.T) {
|
|
||||||
ptr := func(s string) *string { return &s }
|
|
||||||
|
|
||||||
for _, test := range []struct {
|
|
||||||
quote string
|
|
||||||
force bool
|
|
||||||
want *string
|
|
||||||
}{
|
|
||||||
{"", false, nil},
|
|
||||||
{"", true, ptr("")},
|
|
||||||
{"-", false, ptr("-")},
|
|
||||||
{"-", true, ptr("")},
|
|
||||||
} {
|
|
||||||
o := CSVOptions{
|
|
||||||
Quote: test.quote,
|
|
||||||
ForceZeroQuote: test.force,
|
|
||||||
}
|
|
||||||
got := o.quote()
|
|
||||||
if (got == nil) != (test.want == nil) {
|
|
||||||
t.Errorf("%+v\ngot %v\nwant %v", test, pretty.Value(got), pretty.Value(test.want))
|
|
||||||
}
|
|
||||||
if got != nil && test.want != nil && *got != *test.want {
|
|
||||||
t.Errorf("%+v: got %q, want %q", test, *got, *test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQualifier(t *testing.T) {
|
|
||||||
b := BigtableColumn{Qualifier: "a"}
|
|
||||||
q := b.toBQ()
|
|
||||||
if q.QualifierString != b.Qualifier || q.QualifierEncoded != "" {
|
|
||||||
t.Errorf("got (%q, %q), want (%q, %q)",
|
|
||||||
q.QualifierString, q.QualifierEncoded, b.Qualifier, "")
|
|
||||||
}
|
|
||||||
b2, err := bqToBigtableColumn(q)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, want := b2.Qualifier, b.Qualifier; got != want {
|
|
||||||
t.Errorf("got %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
invalidUTF8 = "\xDF\xFF"
|
|
||||||
invalidEncoded = "3/8"
|
|
||||||
)
|
|
||||||
b = BigtableColumn{Qualifier: invalidUTF8}
|
|
||||||
q = b.toBQ()
|
|
||||||
if q.QualifierString != "" || q.QualifierEncoded != invalidEncoded {
|
|
||||||
t.Errorf("got (%q, %q), want (%q, %q)",
|
|
||||||
q.QualifierString, "", b.Qualifier, invalidEncoded)
|
|
||||||
}
|
|
||||||
b2, err = bqToBigtableColumn(q)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, want := b2.Qualifier, b.Qualifier; got != want {
|
|
||||||
t.Errorf("got %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
109
vendor/cloud.google.com/go/bigquery/extract.go
generated
vendored
109
vendor/cloud.google.com/go/bigquery/extract.go
generated
vendored
@ -1,109 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ExtractConfig holds the configuration for an extract job.
|
|
||||||
type ExtractConfig struct {
|
|
||||||
// Src is the table from which data will be extracted.
|
|
||||||
Src *Table
|
|
||||||
|
|
||||||
// Dst is the destination into which the data will be extracted.
|
|
||||||
Dst *GCSReference
|
|
||||||
|
|
||||||
// DisableHeader disables the printing of a header row in exported data.
|
|
||||||
DisableHeader bool
|
|
||||||
|
|
||||||
// The labels associated with this job.
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *ExtractConfig) toBQ() *bq.JobConfiguration {
|
|
||||||
var printHeader *bool
|
|
||||||
if e.DisableHeader {
|
|
||||||
f := false
|
|
||||||
printHeader = &f
|
|
||||||
}
|
|
||||||
return &bq.JobConfiguration{
|
|
||||||
Labels: e.Labels,
|
|
||||||
Extract: &bq.JobConfigurationExtract{
|
|
||||||
DestinationUris: append([]string{}, e.Dst.URIs...),
|
|
||||||
Compression: string(e.Dst.Compression),
|
|
||||||
DestinationFormat: string(e.Dst.DestinationFormat),
|
|
||||||
FieldDelimiter: e.Dst.FieldDelimiter,
|
|
||||||
SourceTable: e.Src.toBQ(),
|
|
||||||
PrintHeader: printHeader,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToExtractConfig(q *bq.JobConfiguration, c *Client) *ExtractConfig {
|
|
||||||
qe := q.Extract
|
|
||||||
return &ExtractConfig{
|
|
||||||
Labels: q.Labels,
|
|
||||||
Dst: &GCSReference{
|
|
||||||
URIs: qe.DestinationUris,
|
|
||||||
Compression: Compression(qe.Compression),
|
|
||||||
DestinationFormat: DataFormat(qe.DestinationFormat),
|
|
||||||
FileConfig: FileConfig{
|
|
||||||
CSVOptions: CSVOptions{
|
|
||||||
FieldDelimiter: qe.FieldDelimiter,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
DisableHeader: qe.PrintHeader != nil && !*qe.PrintHeader,
|
|
||||||
Src: bqToTable(qe.SourceTable, c),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An Extractor extracts data from a BigQuery table into Google Cloud Storage.
|
|
||||||
type Extractor struct {
|
|
||||||
JobIDConfig
|
|
||||||
ExtractConfig
|
|
||||||
c *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExtractorTo returns an Extractor which can be used to extract data from a
|
|
||||||
// BigQuery table into Google Cloud Storage.
|
|
||||||
// The returned Extractor may optionally be further configured before its Run method is called.
|
|
||||||
func (t *Table) ExtractorTo(dst *GCSReference) *Extractor {
|
|
||||||
return &Extractor{
|
|
||||||
c: t.c,
|
|
||||||
ExtractConfig: ExtractConfig{
|
|
||||||
Src: t,
|
|
||||||
Dst: dst,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run initiates an extract job.
|
|
||||||
func (e *Extractor) Run(ctx context.Context) (j *Job, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Extractor.Run")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
return e.c.insertJob(ctx, e.newJob(), nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Extractor) newJob() *bq.Job {
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: e.JobIDConfig.createJobRef(e.c),
|
|
||||||
Configuration: e.ExtractConfig.toBQ(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
118
vendor/cloud.google.com/go/bigquery/extract_test.go
generated
vendored
118
vendor/cloud.google.com/go/bigquery/extract_test.go
generated
vendored
@ -1,118 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func defaultExtractJob() *bq.Job {
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
|
|
||||||
Configuration: &bq.JobConfiguration{
|
|
||||||
Extract: &bq.JobConfigurationExtract{
|
|
||||||
SourceTable: &bq.TableReference{
|
|
||||||
ProjectId: "client-project-id",
|
|
||||||
DatasetId: "dataset-id",
|
|
||||||
TableId: "table-id",
|
|
||||||
},
|
|
||||||
DestinationUris: []string{"uri"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultGCS() *GCSReference {
|
|
||||||
return &GCSReference{
|
|
||||||
URIs: []string{"uri"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExtract(t *testing.T) {
|
|
||||||
defer fixRandomID("RANDOM")()
|
|
||||||
c := &Client{
|
|
||||||
projectID: "client-project-id",
|
|
||||||
}
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
dst *GCSReference
|
|
||||||
src *Table
|
|
||||||
config ExtractConfig
|
|
||||||
want *bq.Job
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
dst: defaultGCS(),
|
|
||||||
src: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
want: defaultExtractJob(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: defaultGCS(),
|
|
||||||
src: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
config: ExtractConfig{
|
|
||||||
DisableHeader: true,
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultExtractJob()
|
|
||||||
j.Configuration.Labels = map[string]string{"a": "b"}
|
|
||||||
f := false
|
|
||||||
j.Configuration.Extract.PrintHeader = &f
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.Compression = Gzip
|
|
||||||
g.DestinationFormat = JSON
|
|
||||||
g.FieldDelimiter = "\t"
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
src: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultExtractJob()
|
|
||||||
j.Configuration.Extract.Compression = "GZIP"
|
|
||||||
j.Configuration.Extract.DestinationFormat = "NEWLINE_DELIMITED_JSON"
|
|
||||||
j.Configuration.Extract.FieldDelimiter = "\t"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, tc := range testCases {
|
|
||||||
ext := tc.src.ExtractorTo(tc.dst)
|
|
||||||
tc.config.Src = ext.Src
|
|
||||||
tc.config.Dst = ext.Dst
|
|
||||||
ext.ExtractConfig = tc.config
|
|
||||||
got := ext.newJob()
|
|
||||||
checkJob(t, i, got, tc.want)
|
|
||||||
|
|
||||||
jc, err := bqToJobConfig(got.Configuration, c)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("#%d: %v", i, err)
|
|
||||||
}
|
|
||||||
diff := testutil.Diff(jc, &ext.ExtractConfig,
|
|
||||||
cmp.AllowUnexported(Table{}, Client{}))
|
|
||||||
if diff != "" {
|
|
||||||
t.Errorf("#%d: (got=-, want=+:\n%s", i, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
135
vendor/cloud.google.com/go/bigquery/file.go
generated
vendored
135
vendor/cloud.google.com/go/bigquery/file.go
generated
vendored
@ -1,135 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A ReaderSource is a source for a load operation that gets
|
|
||||||
// data from an io.Reader.
|
|
||||||
//
|
|
||||||
// When a ReaderSource is part of a LoadConfig obtained via Job.Config,
|
|
||||||
// its internal io.Reader will be nil, so it cannot be used for a
|
|
||||||
// subsequent load operation.
|
|
||||||
type ReaderSource struct {
|
|
||||||
r io.Reader
|
|
||||||
FileConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewReaderSource creates a ReaderSource from an io.Reader. You may
|
|
||||||
// optionally configure properties on the ReaderSource that describe the
|
|
||||||
// data being read, before passing it to Table.LoaderFrom.
|
|
||||||
func NewReaderSource(r io.Reader) *ReaderSource {
|
|
||||||
return &ReaderSource{r: r}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ReaderSource) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader {
|
|
||||||
r.FileConfig.populateLoadConfig(lc)
|
|
||||||
return r.r
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileConfig contains configuration options that pertain to files, typically
|
|
||||||
// text files that require interpretation to be used as a BigQuery table. A
|
|
||||||
// file may live in Google Cloud Storage (see GCSReference), or it may be
|
|
||||||
// loaded into a table via the Table.LoaderFromReader.
|
|
||||||
type FileConfig struct {
|
|
||||||
// SourceFormat is the format of the data to be read.
|
|
||||||
// Allowed values are: CSV, Avro, Parquet, JSON, DatastoreBackup. The default is CSV.
|
|
||||||
SourceFormat DataFormat
|
|
||||||
|
|
||||||
// Indicates if we should automatically infer the options and
|
|
||||||
// schema for CSV and JSON sources.
|
|
||||||
AutoDetect bool
|
|
||||||
|
|
||||||
// MaxBadRecords is the maximum number of bad records that will be ignored
|
|
||||||
// when reading data.
|
|
||||||
MaxBadRecords int64
|
|
||||||
|
|
||||||
// IgnoreUnknownValues causes values not matching the schema to be
|
|
||||||
// tolerated. Unknown values are ignored. For CSV this ignores extra values
|
|
||||||
// at the end of a line. For JSON this ignores named values that do not
|
|
||||||
// match any column name. If this field is not set, records containing
|
|
||||||
// unknown values are treated as bad records. The MaxBadRecords field can
|
|
||||||
// be used to customize how bad records are handled.
|
|
||||||
IgnoreUnknownValues bool
|
|
||||||
|
|
||||||
// Schema describes the data. It is required when reading CSV or JSON data,
|
|
||||||
// unless the data is being loaded into a table that already exists.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// Additional options for CSV files.
|
|
||||||
CSVOptions
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
|
|
||||||
conf.SkipLeadingRows = fc.SkipLeadingRows
|
|
||||||
conf.SourceFormat = string(fc.SourceFormat)
|
|
||||||
conf.Autodetect = fc.AutoDetect
|
|
||||||
conf.AllowJaggedRows = fc.AllowJaggedRows
|
|
||||||
conf.AllowQuotedNewlines = fc.AllowQuotedNewlines
|
|
||||||
conf.Encoding = string(fc.Encoding)
|
|
||||||
conf.FieldDelimiter = fc.FieldDelimiter
|
|
||||||
conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
|
|
||||||
conf.MaxBadRecords = fc.MaxBadRecords
|
|
||||||
if fc.Schema != nil {
|
|
||||||
conf.Schema = fc.Schema.toBQ()
|
|
||||||
}
|
|
||||||
conf.Quote = fc.quote()
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqPopulateFileConfig(conf *bq.JobConfigurationLoad, fc *FileConfig) {
|
|
||||||
fc.SourceFormat = DataFormat(conf.SourceFormat)
|
|
||||||
fc.AutoDetect = conf.Autodetect
|
|
||||||
fc.MaxBadRecords = conf.MaxBadRecords
|
|
||||||
fc.IgnoreUnknownValues = conf.IgnoreUnknownValues
|
|
||||||
fc.Schema = bqToSchema(conf.Schema)
|
|
||||||
fc.SkipLeadingRows = conf.SkipLeadingRows
|
|
||||||
fc.AllowJaggedRows = conf.AllowJaggedRows
|
|
||||||
fc.AllowQuotedNewlines = conf.AllowQuotedNewlines
|
|
||||||
fc.Encoding = Encoding(conf.Encoding)
|
|
||||||
fc.FieldDelimiter = conf.FieldDelimiter
|
|
||||||
fc.CSVOptions.setQuote(conf.Quote)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fc *FileConfig) populateExternalDataConfig(conf *bq.ExternalDataConfiguration) {
|
|
||||||
format := fc.SourceFormat
|
|
||||||
if format == "" {
|
|
||||||
// Format must be explicitly set for external data sources.
|
|
||||||
format = CSV
|
|
||||||
}
|
|
||||||
conf.Autodetect = fc.AutoDetect
|
|
||||||
conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
|
|
||||||
conf.MaxBadRecords = fc.MaxBadRecords
|
|
||||||
conf.SourceFormat = string(format)
|
|
||||||
if fc.Schema != nil {
|
|
||||||
conf.Schema = fc.Schema.toBQ()
|
|
||||||
}
|
|
||||||
if format == CSV {
|
|
||||||
fc.CSVOptions.populateExternalDataConfig(conf)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encoding specifies the character encoding of data to be loaded into BigQuery.
|
|
||||||
// See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding
|
|
||||||
// for more details about how this is used.
|
|
||||||
type Encoding string
|
|
||||||
|
|
||||||
const (
|
|
||||||
UTF_8 Encoding = "UTF-8"
|
|
||||||
ISO_8859_1 Encoding = "ISO-8859-1"
|
|
||||||
)
|
|
||||||
98
vendor/cloud.google.com/go/bigquery/file_test.go
generated
vendored
98
vendor/cloud.google.com/go/bigquery/file_test.go
generated
vendored
@ -1,98 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/pretty"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
hyphen = "-"
|
|
||||||
fc = FileConfig{
|
|
||||||
SourceFormat: CSV,
|
|
||||||
AutoDetect: true,
|
|
||||||
MaxBadRecords: 7,
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
Schema: Schema{
|
|
||||||
stringFieldSchema(),
|
|
||||||
nestedFieldSchema(),
|
|
||||||
},
|
|
||||||
CSVOptions: CSVOptions{
|
|
||||||
Quote: hyphen,
|
|
||||||
FieldDelimiter: "\t",
|
|
||||||
SkipLeadingRows: 8,
|
|
||||||
AllowJaggedRows: true,
|
|
||||||
AllowQuotedNewlines: true,
|
|
||||||
Encoding: UTF_8,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFileConfigPopulateLoadConfig(t *testing.T) {
|
|
||||||
want := &bq.JobConfigurationLoad{
|
|
||||||
SourceFormat: "CSV",
|
|
||||||
FieldDelimiter: "\t",
|
|
||||||
SkipLeadingRows: 8,
|
|
||||||
AllowJaggedRows: true,
|
|
||||||
AllowQuotedNewlines: true,
|
|
||||||
Autodetect: true,
|
|
||||||
Encoding: "UTF-8",
|
|
||||||
MaxBadRecords: 7,
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
Schema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqStringFieldSchema(),
|
|
||||||
bqNestedFieldSchema(),
|
|
||||||
}},
|
|
||||||
Quote: &hyphen,
|
|
||||||
}
|
|
||||||
got := &bq.JobConfigurationLoad{}
|
|
||||||
fc.populateLoadConfig(got)
|
|
||||||
if !testutil.Equal(got, want) {
|
|
||||||
t.Errorf("got:\n%v\nwant:\n%v", pretty.Value(got), pretty.Value(want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileConfigPopulateExternalDataConfig(t *testing.T) {
|
|
||||||
got := &bq.ExternalDataConfiguration{}
|
|
||||||
fc.populateExternalDataConfig(got)
|
|
||||||
|
|
||||||
want := &bq.ExternalDataConfiguration{
|
|
||||||
SourceFormat: "CSV",
|
|
||||||
Autodetect: true,
|
|
||||||
MaxBadRecords: 7,
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
Schema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqStringFieldSchema(),
|
|
||||||
bqNestedFieldSchema(),
|
|
||||||
}},
|
|
||||||
CsvOptions: &bq.CsvOptions{
|
|
||||||
AllowJaggedRows: true,
|
|
||||||
AllowQuotedNewlines: true,
|
|
||||||
Encoding: "UTF-8",
|
|
||||||
FieldDelimiter: "\t",
|
|
||||||
Quote: &hyphen,
|
|
||||||
SkipLeadingRows: 8,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Errorf("got=-, want=+:\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
73
vendor/cloud.google.com/go/bigquery/gcs.go
generated
vendored
73
vendor/cloud.google.com/go/bigquery/gcs.go
generated
vendored
@ -1,73 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute
|
|
||||||
// an input or output to a BigQuery operation.
|
|
||||||
type GCSReference struct {
|
|
||||||
// URIs refer to Google Cloud Storage objects.
|
|
||||||
URIs []string
|
|
||||||
|
|
||||||
FileConfig
|
|
||||||
|
|
||||||
// DestinationFormat is the format to use when writing exported files.
|
|
||||||
// Allowed values are: CSV, Avro, JSON. The default is CSV.
|
|
||||||
// CSV is not supported for tables with nested or repeated fields.
|
|
||||||
DestinationFormat DataFormat
|
|
||||||
|
|
||||||
// Compression specifies the type of compression to apply when writing data
|
|
||||||
// to Google Cloud Storage, or using this GCSReference as an ExternalData
|
|
||||||
// source with CSV or JSON SourceFormat. Default is None.
|
|
||||||
Compression Compression
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination.
|
|
||||||
// In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object.
|
|
||||||
// Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided.
|
|
||||||
// Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name.
|
|
||||||
// For more information about the treatment of wildcards and multiple URIs,
|
|
||||||
// see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
|
|
||||||
func NewGCSReference(uri ...string) *GCSReference {
|
|
||||||
return &GCSReference{URIs: uri}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compression is the type of compression to apply when writing data to Google Cloud Storage.
|
|
||||||
type Compression string
|
|
||||||
|
|
||||||
const (
|
|
||||||
None Compression = "NONE"
|
|
||||||
Gzip Compression = "GZIP"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (gcs *GCSReference) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader {
|
|
||||||
lc.SourceUris = gcs.URIs
|
|
||||||
gcs.FileConfig.populateLoadConfig(lc)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (gcs *GCSReference) toBQ() bq.ExternalDataConfiguration {
|
|
||||||
conf := bq.ExternalDataConfiguration{
|
|
||||||
Compression: string(gcs.Compression),
|
|
||||||
SourceUris: append([]string{}, gcs.URIs...),
|
|
||||||
}
|
|
||||||
gcs.FileConfig.populateExternalDataConfig(&conf)
|
|
||||||
return conf
|
|
||||||
}
|
|
||||||
1918
vendor/cloud.google.com/go/bigquery/integration_test.go
generated
vendored
1918
vendor/cloud.google.com/go/bigquery/integration_test.go
generated
vendored
File diff suppressed because it is too large
Load Diff
215
vendor/cloud.google.com/go/bigquery/iterator.go
generated
vendored
215
vendor/cloud.google.com/go/bigquery/iterator.go
generated
vendored
@ -1,215 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
func newRowIterator(ctx context.Context, t *Table, pf pageFetcher) *RowIterator {
|
|
||||||
it := &RowIterator{
|
|
||||||
ctx: ctx,
|
|
||||||
table: t,
|
|
||||||
pf: pf,
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
|
||||||
it.fetch,
|
|
||||||
func() int { return len(it.rows) },
|
|
||||||
func() interface{} { r := it.rows; it.rows = nil; return r })
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// A RowIterator provides access to the result of a BigQuery lookup.
|
|
||||||
type RowIterator struct {
|
|
||||||
ctx context.Context
|
|
||||||
table *Table
|
|
||||||
pf pageFetcher
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
|
|
||||||
// StartIndex can be set before the first call to Next. If PageInfo().Token
|
|
||||||
// is also set, StartIndex is ignored.
|
|
||||||
StartIndex uint64
|
|
||||||
|
|
||||||
// The schema of the table. Available after the first call to Next.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// The total number of rows in the result. Available after the first call to Next.
|
|
||||||
// May be zero just after rows were inserted.
|
|
||||||
TotalRows uint64
|
|
||||||
|
|
||||||
rows [][]Value
|
|
||||||
structLoader structLoader // used to populate a pointer to a struct
|
|
||||||
}
|
|
||||||
|
|
||||||
// Next loads the next row into dst. Its return value is iterator.Done if there
|
|
||||||
// are no more results. Once Next returns iterator.Done, all subsequent calls
|
|
||||||
// will return iterator.Done.
|
|
||||||
//
|
|
||||||
// dst may implement ValueLoader, or may be a *[]Value, *map[string]Value, or struct pointer.
|
|
||||||
//
|
|
||||||
// If dst is a *[]Value, it will be set to to new []Value whose i'th element
|
|
||||||
// will be populated with the i'th column of the row.
|
|
||||||
//
|
|
||||||
// If dst is a *map[string]Value, a new map will be created if dst is nil. Then
|
|
||||||
// for each schema column name, the map key of that name will be set to the column's
|
|
||||||
// value. STRUCT types (RECORD types or nested schemas) become nested maps.
|
|
||||||
//
|
|
||||||
// If dst is pointer to a struct, each column in the schema will be matched
|
|
||||||
// with an exported field of the struct that has the same name, ignoring case.
|
|
||||||
// Unmatched schema columns and struct fields will be ignored.
|
|
||||||
//
|
|
||||||
// Each BigQuery column type corresponds to one or more Go types; a matching struct
|
|
||||||
// field must be of the correct type. The correspondences are:
|
|
||||||
//
|
|
||||||
// STRING string
|
|
||||||
// BOOL bool
|
|
||||||
// INTEGER int, int8, int16, int32, int64, uint8, uint16, uint32
|
|
||||||
// FLOAT float32, float64
|
|
||||||
// BYTES []byte
|
|
||||||
// TIMESTAMP time.Time
|
|
||||||
// DATE civil.Date
|
|
||||||
// TIME civil.Time
|
|
||||||
// DATETIME civil.DateTime
|
|
||||||
//
|
|
||||||
// A repeated field corresponds to a slice or array of the element type. A STRUCT
|
|
||||||
// type (RECORD or nested schema) corresponds to a nested struct or struct pointer.
|
|
||||||
// All calls to Next on the same iterator must use the same struct type.
|
|
||||||
//
|
|
||||||
// It is an error to attempt to read a BigQuery NULL value into a struct field,
|
|
||||||
// unless the field is of type []byte or is one of the special Null types: NullInt64,
|
|
||||||
// NullFloat64, NullBool, NullString, NullTimestamp, NullDate, NullTime or
|
|
||||||
// NullDateTime. You can also use a *[]Value or *map[string]Value to read from a
|
|
||||||
// table with NULLs.
|
|
||||||
func (it *RowIterator) Next(dst interface{}) error {
|
|
||||||
var vl ValueLoader
|
|
||||||
switch dst := dst.(type) {
|
|
||||||
case ValueLoader:
|
|
||||||
vl = dst
|
|
||||||
case *[]Value:
|
|
||||||
vl = (*valueList)(dst)
|
|
||||||
case *map[string]Value:
|
|
||||||
vl = (*valueMap)(dst)
|
|
||||||
default:
|
|
||||||
if !isStructPtr(dst) {
|
|
||||||
return fmt.Errorf("bigquery: cannot convert %T to ValueLoader (need pointer to []Value, map[string]Value, or struct)", dst)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
row := it.rows[0]
|
|
||||||
it.rows = it.rows[1:]
|
|
||||||
|
|
||||||
if vl == nil {
|
|
||||||
// This can only happen if dst is a pointer to a struct. We couldn't
|
|
||||||
// set vl above because we need the schema.
|
|
||||||
if err := it.structLoader.set(dst, it.Schema); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
vl = &it.structLoader
|
|
||||||
}
|
|
||||||
return vl.Load(row, it.Schema)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isStructPtr(x interface{}) bool {
|
|
||||||
t := reflect.TypeOf(x)
|
|
||||||
return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
|
|
||||||
}
|
|
||||||
|
|
||||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
|
||||||
func (it *RowIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
|
||||||
|
|
||||||
func (it *RowIterator) fetch(pageSize int, pageToken string) (string, error) {
|
|
||||||
res, err := it.pf(it.ctx, it.table, it.Schema, it.StartIndex, int64(pageSize), pageToken)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.rows = append(it.rows, res.rows...)
|
|
||||||
it.Schema = res.schema
|
|
||||||
it.TotalRows = res.totalRows
|
|
||||||
return res.pageToken, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// A pageFetcher returns a page of rows from a destination table.
|
|
||||||
type pageFetcher func(ctx context.Context, _ *Table, _ Schema, startIndex uint64, pageSize int64, pageToken string) (*fetchPageResult, error)
|
|
||||||
|
|
||||||
type fetchPageResult struct {
|
|
||||||
pageToken string
|
|
||||||
rows [][]Value
|
|
||||||
totalRows uint64
|
|
||||||
schema Schema
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchPage gets a page of rows from t.
|
|
||||||
func fetchPage(ctx context.Context, t *Table, schema Schema, startIndex uint64, pageSize int64, pageToken string) (*fetchPageResult, error) {
|
|
||||||
// Fetch the table schema in the background, if necessary.
|
|
||||||
errc := make(chan error, 1)
|
|
||||||
if schema != nil {
|
|
||||||
errc <- nil
|
|
||||||
} else {
|
|
||||||
go func() {
|
|
||||||
var bqt *bq.Table
|
|
||||||
err := runWithRetry(ctx, func() (err error) {
|
|
||||||
bqt, err = t.c.bqs.Tables.Get(t.ProjectID, t.DatasetID, t.TableID).
|
|
||||||
Fields("schema").
|
|
||||||
Context(ctx).
|
|
||||||
Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if err == nil && bqt.Schema != nil {
|
|
||||||
schema = bqToSchema(bqt.Schema)
|
|
||||||
}
|
|
||||||
errc <- err
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
call := t.c.bqs.Tabledata.List(t.ProjectID, t.DatasetID, t.TableID)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if pageToken != "" {
|
|
||||||
call.PageToken(pageToken)
|
|
||||||
} else {
|
|
||||||
call.StartIndex(startIndex)
|
|
||||||
}
|
|
||||||
if pageSize > 0 {
|
|
||||||
call.MaxResults(pageSize)
|
|
||||||
}
|
|
||||||
var res *bq.TableDataList
|
|
||||||
err := runWithRetry(ctx, func() (err error) {
|
|
||||||
res, err = call.Context(ctx).Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
err = <-errc
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rows, err := convertRows(res.Rows, schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &fetchPageResult{
|
|
||||||
pageToken: res.PageToken,
|
|
||||||
rows: rows,
|
|
||||||
totalRows: uint64(res.TotalRows),
|
|
||||||
schema: schema,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
363
vendor/cloud.google.com/go/bigquery/iterator_test.go
generated
vendored
363
vendor/cloud.google.com/go/bigquery/iterator_test.go
generated
vendored
@ -1,363 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
type fetchResponse struct {
|
|
||||||
result *fetchPageResult // The result to return.
|
|
||||||
err error // The error to return.
|
|
||||||
}
|
|
||||||
|
|
||||||
// pageFetcherStub services fetch requests by returning data from an in-memory list of values.
|
|
||||||
type pageFetcherStub struct {
|
|
||||||
fetchResponses map[string]fetchResponse
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (pf *pageFetcherStub) fetchPage(ctx context.Context, _ *Table, _ Schema, _ uint64, _ int64, pageToken string) (*fetchPageResult, error) {
|
|
||||||
call, ok := pf.fetchResponses[pageToken]
|
|
||||||
if !ok {
|
|
||||||
pf.err = fmt.Errorf("Unexpected page token: %q", pageToken)
|
|
||||||
}
|
|
||||||
return call.result, call.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIterator(t *testing.T) {
|
|
||||||
var (
|
|
||||||
iiSchema = Schema{
|
|
||||||
{Type: IntegerFieldType},
|
|
||||||
{Type: IntegerFieldType},
|
|
||||||
}
|
|
||||||
siSchema = Schema{
|
|
||||||
{Type: StringFieldType},
|
|
||||||
{Type: IntegerFieldType},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
fetchFailure := errors.New("fetch failure")
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
desc string
|
|
||||||
pageToken string
|
|
||||||
fetchResponses map[string]fetchResponse
|
|
||||||
want [][]Value
|
|
||||||
wantErr error
|
|
||||||
wantSchema Schema
|
|
||||||
wantTotalRows uint64
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
desc: "Iteration over single empty page",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{},
|
|
||||||
schema: Schema{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{},
|
|
||||||
wantSchema: Schema{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "Iteration over single page",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
totalRows: 4,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
wantSchema: iiSchema,
|
|
||||||
wantTotalRows: 4,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "Iteration over single page with different schema",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{"1", 2}, {"11", 12}},
|
|
||||||
schema: siSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{"1", 2}, {"11", 12}},
|
|
||||||
wantSchema: siSchema,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "Iteration over two pages",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "a",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
totalRows: 4,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"a": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
schema: iiSchema,
|
|
||||||
totalRows: 4,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
|
|
||||||
wantSchema: iiSchema,
|
|
||||||
wantTotalRows: 4,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "Server response includes empty page",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "a",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"a": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "b",
|
|
||||||
rows: [][]Value{},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"b": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
|
|
||||||
wantSchema: iiSchema,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "Fetch error",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "a",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"a": {
|
|
||||||
// We returns some data from this fetch, but also an error.
|
|
||||||
// So the end result should include only data from the previous fetch.
|
|
||||||
err: fetchFailure,
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "b",
|
|
||||||
rows: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
wantErr: fetchFailure,
|
|
||||||
wantSchema: iiSchema,
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
desc: "Skip over an entire page",
|
|
||||||
pageToken: "a",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "a",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"a": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
wantSchema: iiSchema,
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
desc: "Skip beyond all data",
|
|
||||||
pageToken: "b",
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "a",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"a": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "b",
|
|
||||||
rows: [][]Value{{101, 102}, {111, 112}},
|
|
||||||
schema: iiSchema,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"b": {
|
|
||||||
result: &fetchPageResult{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
// In this test case, Next will return false on its first call,
|
|
||||||
// so we won't even attempt to call Get.
|
|
||||||
want: [][]Value{},
|
|
||||||
wantSchema: Schema{},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
pf := &pageFetcherStub{
|
|
||||||
fetchResponses: tc.fetchResponses,
|
|
||||||
}
|
|
||||||
it := newRowIterator(context.Background(), nil, pf.fetchPage)
|
|
||||||
it.PageInfo().Token = tc.pageToken
|
|
||||||
values, schema, totalRows, err := consumeRowIterator(it)
|
|
||||||
if err != tc.wantErr {
|
|
||||||
t.Fatalf("%s: got %v, want %v", tc.desc, err, tc.wantErr)
|
|
||||||
}
|
|
||||||
if (len(values) != 0 || len(tc.want) != 0) && !testutil.Equal(values, tc.want) {
|
|
||||||
t.Errorf("%s: values:\ngot: %v\nwant:%v", tc.desc, values, tc.want)
|
|
||||||
}
|
|
||||||
if (len(schema) != 0 || len(tc.wantSchema) != 0) && !testutil.Equal(schema, tc.wantSchema) {
|
|
||||||
t.Errorf("%s: iterator.Schema:\ngot: %v\nwant: %v", tc.desc, schema, tc.wantSchema)
|
|
||||||
}
|
|
||||||
if totalRows != tc.wantTotalRows {
|
|
||||||
t.Errorf("%s: totalRows: got %d, want %d", tc.desc, totalRows, tc.wantTotalRows)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// consumeRowIterator reads the schema and all values from a RowIterator and returns them.
|
|
||||||
func consumeRowIterator(it *RowIterator) ([][]Value, Schema, uint64, error) {
|
|
||||||
var (
|
|
||||||
got [][]Value
|
|
||||||
schema Schema
|
|
||||||
totalRows uint64
|
|
||||||
)
|
|
||||||
for {
|
|
||||||
var vls []Value
|
|
||||||
err := it.Next(&vls)
|
|
||||||
if err == iterator.Done {
|
|
||||||
return got, schema, totalRows, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return got, schema, totalRows, err
|
|
||||||
}
|
|
||||||
got = append(got, vls)
|
|
||||||
schema = it.Schema
|
|
||||||
totalRows = it.TotalRows
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNextDuringErrorState(t *testing.T) {
|
|
||||||
pf := &pageFetcherStub{
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {err: errors.New("bang")},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
it := newRowIterator(context.Background(), nil, pf.fetchPage)
|
|
||||||
var vals []Value
|
|
||||||
if err := it.Next(&vals); err == nil {
|
|
||||||
t.Errorf("Expected error after calling Next")
|
|
||||||
}
|
|
||||||
if err := it.Next(&vals); err == nil {
|
|
||||||
t.Errorf("Expected error calling Next again when iterator has a non-nil error.")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNextAfterFinished(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
fetchResponses map[string]fetchResponse
|
|
||||||
want [][]Value
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
fetchResponses: map[string]fetchResponse{
|
|
||||||
"": {
|
|
||||||
result: &fetchPageResult{
|
|
||||||
pageToken: "",
|
|
||||||
rows: [][]Value{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: [][]Value{},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
pf := &pageFetcherStub{
|
|
||||||
fetchResponses: tc.fetchResponses,
|
|
||||||
}
|
|
||||||
it := newRowIterator(context.Background(), nil, pf.fetchPage)
|
|
||||||
|
|
||||||
values, _, _, err := consumeRowIterator(it)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if (len(values) != 0 || len(tc.want) != 0) && !testutil.Equal(values, tc.want) {
|
|
||||||
t.Errorf("values: got:\n%v\nwant:\n%v", values, tc.want)
|
|
||||||
}
|
|
||||||
// Try calling Get again.
|
|
||||||
var vals []Value
|
|
||||||
if err := it.Next(&vals); err != iterator.Done {
|
|
||||||
t.Errorf("Expected Done calling Next when there are no more values")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIteratorNextTypes(t *testing.T) {
|
|
||||||
it := newRowIterator(context.Background(), nil, nil)
|
|
||||||
for _, v := range []interface{}{3, "s", []int{}, &[]int{},
|
|
||||||
map[string]Value{}, &map[string]interface{}{},
|
|
||||||
struct{}{},
|
|
||||||
} {
|
|
||||||
if err := it.Next(v); err == nil {
|
|
||||||
t.Errorf("%v: want error, got nil", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
822
vendor/cloud.google.com/go/bigquery/job.go
generated
vendored
822
vendor/cloud.google.com/go/bigquery/job.go
generated
vendored
@ -1,822 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal"
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
gax "github.com/googleapis/gax-go"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
"google.golang.org/api/googleapi"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A Job represents an operation which has been submitted to BigQuery for processing.
|
|
||||||
type Job struct {
|
|
||||||
c *Client
|
|
||||||
projectID string
|
|
||||||
jobID string
|
|
||||||
location string
|
|
||||||
|
|
||||||
config *bq.JobConfiguration
|
|
||||||
lastStatus *JobStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobFromID creates a Job which refers to an existing BigQuery job. The job
|
|
||||||
// need not have been created by this package. For example, the job may have
|
|
||||||
// been created in the BigQuery console.
|
|
||||||
//
|
|
||||||
// For jobs whose location is other than "US" or "EU", set Client.Location or use
|
|
||||||
// JobFromIDLocation.
|
|
||||||
func (c *Client) JobFromID(ctx context.Context, id string) (*Job, error) {
|
|
||||||
return c.JobFromIDLocation(ctx, id, c.Location)
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobFromIDLocation creates a Job which refers to an existing BigQuery job. The job
|
|
||||||
// need not have been created by this package (for example, it may have
|
|
||||||
// been created in the BigQuery console), but it must exist in the specified location.
|
|
||||||
func (c *Client) JobFromIDLocation(ctx context.Context, id, location string) (j *Job, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.JobFromIDLocation")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
bqjob, err := c.getJobInternal(ctx, id, location, "configuration", "jobReference", "status", "statistics")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToJob(bqjob, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ID returns the job's ID.
|
|
||||||
func (j *Job) ID() string {
|
|
||||||
return j.jobID
|
|
||||||
}
|
|
||||||
|
|
||||||
// Location returns the job's location.
|
|
||||||
func (j *Job) Location() string {
|
|
||||||
return j.location
|
|
||||||
}
|
|
||||||
|
|
||||||
// State is one of a sequence of states that a Job progresses through as it is processed.
|
|
||||||
type State int
|
|
||||||
|
|
||||||
const (
|
|
||||||
StateUnspecified State = iota // used only as a default in JobIterator
|
|
||||||
Pending
|
|
||||||
Running
|
|
||||||
Done
|
|
||||||
)
|
|
||||||
|
|
||||||
// JobStatus contains the current State of a job, and errors encountered while processing that job.
|
|
||||||
type JobStatus struct {
|
|
||||||
State State
|
|
||||||
|
|
||||||
err error
|
|
||||||
|
|
||||||
// All errors encountered during the running of the job.
|
|
||||||
// Not all Errors are fatal, so errors here do not necessarily mean that the job has completed or was unsuccessful.
|
|
||||||
Errors []*Error
|
|
||||||
|
|
||||||
// Statistics about the job.
|
|
||||||
Statistics *JobStatistics
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobConfig contains configuration information for a job. It is implemented by
|
|
||||||
// *CopyConfig, *ExtractConfig, *LoadConfig and *QueryConfig.
|
|
||||||
type JobConfig interface {
|
|
||||||
isJobConfig()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*CopyConfig) isJobConfig() {}
|
|
||||||
func (*ExtractConfig) isJobConfig() {}
|
|
||||||
func (*LoadConfig) isJobConfig() {}
|
|
||||||
func (*QueryConfig) isJobConfig() {}
|
|
||||||
|
|
||||||
// Config returns the configuration information for j.
|
|
||||||
func (j *Job) Config() (JobConfig, error) {
|
|
||||||
return bqToJobConfig(j.config, j.c)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToJobConfig(q *bq.JobConfiguration, c *Client) (JobConfig, error) {
|
|
||||||
switch {
|
|
||||||
case q == nil:
|
|
||||||
return nil, nil
|
|
||||||
case q.Copy != nil:
|
|
||||||
return bqToCopyConfig(q, c), nil
|
|
||||||
case q.Extract != nil:
|
|
||||||
return bqToExtractConfig(q, c), nil
|
|
||||||
case q.Load != nil:
|
|
||||||
return bqToLoadConfig(q, c), nil
|
|
||||||
case q.Query != nil:
|
|
||||||
return bqToQueryConfig(q, c)
|
|
||||||
default:
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobIDConfig describes how to create an ID for a job.
|
|
||||||
type JobIDConfig struct {
|
|
||||||
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
|
|
||||||
JobID string
|
|
||||||
|
|
||||||
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
|
|
||||||
AddJobIDSuffix bool
|
|
||||||
|
|
||||||
// Location is the location for the job.
|
|
||||||
Location string
|
|
||||||
}
|
|
||||||
|
|
||||||
// createJobRef creates a JobReference.
|
|
||||||
func (j *JobIDConfig) createJobRef(c *Client) *bq.JobReference {
|
|
||||||
// We don't check whether projectID is empty; the server will return an
|
|
||||||
// error when it encounters the resulting JobReference.
|
|
||||||
loc := j.Location
|
|
||||||
if loc == "" { // Use Client.Location as a default.
|
|
||||||
loc = c.Location
|
|
||||||
}
|
|
||||||
jr := &bq.JobReference{ProjectId: c.projectID, Location: loc}
|
|
||||||
if j.JobID == "" {
|
|
||||||
jr.JobId = randomIDFn()
|
|
||||||
} else if j.AddJobIDSuffix {
|
|
||||||
jr.JobId = j.JobID + "-" + randomIDFn()
|
|
||||||
} else {
|
|
||||||
jr.JobId = j.JobID
|
|
||||||
}
|
|
||||||
return jr
|
|
||||||
}
|
|
||||||
|
|
||||||
const alphanum = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|
|
||||||
|
|
||||||
var (
|
|
||||||
rngMu sync.Mutex
|
|
||||||
rng = rand.New(rand.NewSource(time.Now().UnixNano() ^ int64(os.Getpid())))
|
|
||||||
)
|
|
||||||
|
|
||||||
// For testing.
|
|
||||||
var randomIDFn = randomID
|
|
||||||
|
|
||||||
// As of August 2017, the BigQuery service uses 27 alphanumeric characters for
|
|
||||||
// suffixes.
|
|
||||||
const randomIDLen = 27
|
|
||||||
|
|
||||||
func randomID() string {
|
|
||||||
// This is used for both job IDs and insert IDs.
|
|
||||||
var b [randomIDLen]byte
|
|
||||||
rngMu.Lock()
|
|
||||||
for i := 0; i < len(b); i++ {
|
|
||||||
b[i] = alphanum[rng.Intn(len(alphanum))]
|
|
||||||
}
|
|
||||||
rngMu.Unlock()
|
|
||||||
return string(b[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Done reports whether the job has completed.
|
|
||||||
// After Done returns true, the Err method will return an error if the job completed unsuccessfully.
|
|
||||||
func (s *JobStatus) Done() bool {
|
|
||||||
return s.State == Done
|
|
||||||
}
|
|
||||||
|
|
||||||
// Err returns the error that caused the job to complete unsuccessfully (if any).
|
|
||||||
func (s *JobStatus) Err() error {
|
|
||||||
return s.err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Status retrieves the current status of the job from BigQuery. It fails if the Status could not be determined.
|
|
||||||
func (j *Job) Status(ctx context.Context) (js *JobStatus, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Job.Status")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
bqjob, err := j.c.getJobInternal(ctx, j.jobID, j.location, "status", "statistics")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := j.setStatus(bqjob.Status); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
j.setStatistics(bqjob.Statistics, j.c)
|
|
||||||
return j.lastStatus, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LastStatus returns the most recently retrieved status of the job. The status is
|
|
||||||
// retrieved when a new job is created, or when JobFromID or Job.Status is called.
|
|
||||||
// Call Job.Status to get the most up-to-date information about a job.
|
|
||||||
func (j *Job) LastStatus() *JobStatus {
|
|
||||||
return j.lastStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cancel requests that a job be cancelled. This method returns without waiting for
|
|
||||||
// cancellation to take effect. To check whether the job has terminated, use Job.Status.
|
|
||||||
// Cancelled jobs may still incur costs.
|
|
||||||
func (j *Job) Cancel(ctx context.Context) error {
|
|
||||||
// Jobs.Cancel returns a job entity, but the only relevant piece of
|
|
||||||
// data it may contain (the status of the job) is unreliable. From the
|
|
||||||
// docs: "This call will return immediately, and the client will need
|
|
||||||
// to poll for the job status to see if the cancel completed
|
|
||||||
// successfully". So it would be misleading to return a status.
|
|
||||||
call := j.c.bqs.Jobs.Cancel(j.projectID, j.jobID).
|
|
||||||
Location(j.location).
|
|
||||||
Fields(). // We don't need any of the response data.
|
|
||||||
Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
return runWithRetry(ctx, func() error {
|
|
||||||
_, err := call.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait blocks until the job or the context is done. It returns the final status
|
|
||||||
// of the job.
|
|
||||||
// If an error occurs while retrieving the status, Wait returns that error. But
|
|
||||||
// Wait returns nil if the status was retrieved successfully, even if
|
|
||||||
// status.Err() != nil. So callers must check both errors. See the example.
|
|
||||||
func (j *Job) Wait(ctx context.Context) (js *JobStatus, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Job.Wait")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
if j.isQuery() {
|
|
||||||
// We can avoid polling for query jobs.
|
|
||||||
if _, err := j.waitForQuery(ctx, j.projectID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Note: extra RPC even if you just want to wait for the query to finish.
|
|
||||||
js, err := j.Status(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return js, nil
|
|
||||||
}
|
|
||||||
// Non-query jobs must poll.
|
|
||||||
err = internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
|
|
||||||
js, err = j.Status(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return true, err
|
|
||||||
}
|
|
||||||
if js.Done() {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return js, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read fetches the results of a query job.
|
|
||||||
// If j is not a query job, Read returns an error.
|
|
||||||
func (j *Job) Read(ctx context.Context) (ri *RowIterator, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Job.Read")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
return j.read(ctx, j.waitForQuery, fetchPage)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *Job) read(ctx context.Context, waitForQuery func(context.Context, string) (Schema, error), pf pageFetcher) (*RowIterator, error) {
|
|
||||||
if !j.isQuery() {
|
|
||||||
return nil, errors.New("bigquery: cannot read from a non-query job")
|
|
||||||
}
|
|
||||||
destTable := j.config.Query.DestinationTable
|
|
||||||
// The destination table should only be nil if there was a query error.
|
|
||||||
projectID := j.projectID
|
|
||||||
if destTable != nil && projectID != destTable.ProjectId {
|
|
||||||
return nil, fmt.Errorf("bigquery: job project ID is %q, but destination table's is %q", projectID, destTable.ProjectId)
|
|
||||||
}
|
|
||||||
schema, err := waitForQuery(ctx, projectID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if destTable == nil {
|
|
||||||
return nil, errors.New("bigquery: query job missing destination table")
|
|
||||||
}
|
|
||||||
dt := bqToTable(destTable, j.c)
|
|
||||||
it := newRowIterator(ctx, dt, pf)
|
|
||||||
it.Schema = schema
|
|
||||||
return it, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForQuery waits for the query job to complete and returns its schema.
|
|
||||||
func (j *Job) waitForQuery(ctx context.Context, projectID string) (Schema, error) {
|
|
||||||
// Use GetQueryResults only to wait for completion, not to read results.
|
|
||||||
call := j.c.bqs.Jobs.GetQueryResults(projectID, j.jobID).Location(j.location).Context(ctx).MaxResults(0)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
backoff := gax.Backoff{
|
|
||||||
Initial: 1 * time.Second,
|
|
||||||
Multiplier: 2,
|
|
||||||
Max: 60 * time.Second,
|
|
||||||
}
|
|
||||||
var res *bq.GetQueryResultsResponse
|
|
||||||
err := internal.Retry(ctx, backoff, func() (stop bool, err error) {
|
|
||||||
res, err = call.Do()
|
|
||||||
if err != nil {
|
|
||||||
return !retryableError(err), err
|
|
||||||
}
|
|
||||||
if !res.JobComplete { // GetQueryResults may return early without error; retry.
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToSchema(res.Schema), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobStatistics contains statistics about a job.
|
|
||||||
type JobStatistics struct {
|
|
||||||
CreationTime time.Time
|
|
||||||
StartTime time.Time
|
|
||||||
EndTime time.Time
|
|
||||||
TotalBytesProcessed int64
|
|
||||||
|
|
||||||
Details Statistics
|
|
||||||
}
|
|
||||||
|
|
||||||
// Statistics is one of ExtractStatistics, LoadStatistics or QueryStatistics.
|
|
||||||
type Statistics interface {
|
|
||||||
implementsStatistics()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExtractStatistics contains statistics about an extract job.
|
|
||||||
type ExtractStatistics struct {
|
|
||||||
// The number of files per destination URI or URI pattern specified in the
|
|
||||||
// extract configuration. These values will be in the same order as the
|
|
||||||
// URIs specified in the 'destinationUris' field.
|
|
||||||
DestinationURIFileCounts []int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadStatistics contains statistics about a load job.
|
|
||||||
type LoadStatistics struct {
|
|
||||||
// The number of bytes of source data in a load job.
|
|
||||||
InputFileBytes int64
|
|
||||||
|
|
||||||
// The number of source files in a load job.
|
|
||||||
InputFiles int64
|
|
||||||
|
|
||||||
// Size of the loaded data in bytes. Note that while a load job is in the
|
|
||||||
// running state, this value may change.
|
|
||||||
OutputBytes int64
|
|
||||||
|
|
||||||
// The number of rows imported in a load job. Note that while an import job is
|
|
||||||
// in the running state, this value may change.
|
|
||||||
OutputRows int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// QueryStatistics contains statistics about a query job.
|
|
||||||
type QueryStatistics struct {
|
|
||||||
// Billing tier for the job.
|
|
||||||
BillingTier int64
|
|
||||||
|
|
||||||
// Whether the query result was fetched from the query cache.
|
|
||||||
CacheHit bool
|
|
||||||
|
|
||||||
// The type of query statement, if valid.
|
|
||||||
StatementType string
|
|
||||||
|
|
||||||
// Total bytes billed for the job.
|
|
||||||
TotalBytesBilled int64
|
|
||||||
|
|
||||||
// Total bytes processed for the job.
|
|
||||||
TotalBytesProcessed int64
|
|
||||||
|
|
||||||
// Describes execution plan for the query.
|
|
||||||
QueryPlan []*ExplainQueryStage
|
|
||||||
|
|
||||||
// The number of rows affected by a DML statement. Present only for DML
|
|
||||||
// statements INSERT, UPDATE or DELETE.
|
|
||||||
NumDMLAffectedRows int64
|
|
||||||
|
|
||||||
// Describes a timeline of job execution.
|
|
||||||
Timeline []*QueryTimelineSample
|
|
||||||
|
|
||||||
// ReferencedTables: [Output-only, Experimental] Referenced tables for
|
|
||||||
// the job. Queries that reference more than 50 tables will not have a
|
|
||||||
// complete list.
|
|
||||||
ReferencedTables []*Table
|
|
||||||
|
|
||||||
// The schema of the results. Present only for successful dry run of
|
|
||||||
// non-legacy SQL queries.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// Slot-milliseconds consumed by this query job.
|
|
||||||
SlotMillis int64
|
|
||||||
|
|
||||||
// Standard SQL: list of undeclared query parameter names detected during a
|
|
||||||
// dry run validation.
|
|
||||||
UndeclaredQueryParameterNames []string
|
|
||||||
|
|
||||||
// DDL target table.
|
|
||||||
DDLTargetTable *Table
|
|
||||||
|
|
||||||
// DDL Operation performed on the target table. Used to report how the
|
|
||||||
// query impacted the DDL target table.
|
|
||||||
DDLOperationPerformed string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExplainQueryStage describes one stage of a query.
|
|
||||||
type ExplainQueryStage struct {
|
|
||||||
// CompletedParallelInputs: Number of parallel input segments completed.
|
|
||||||
CompletedParallelInputs int64
|
|
||||||
|
|
||||||
// ComputeAvg: Duration the average shard spent on CPU-bound tasks.
|
|
||||||
ComputeAvg time.Duration
|
|
||||||
|
|
||||||
// ComputeMax: Duration the slowest shard spent on CPU-bound tasks.
|
|
||||||
ComputeMax time.Duration
|
|
||||||
|
|
||||||
// Relative amount of the total time the average shard spent on CPU-bound tasks.
|
|
||||||
ComputeRatioAvg float64
|
|
||||||
|
|
||||||
// Relative amount of the total time the slowest shard spent on CPU-bound tasks.
|
|
||||||
ComputeRatioMax float64
|
|
||||||
|
|
||||||
// EndTime: Stage end time.
|
|
||||||
EndTime time.Time
|
|
||||||
|
|
||||||
// Unique ID for stage within plan.
|
|
||||||
ID int64
|
|
||||||
|
|
||||||
// InputStages: IDs for stages that are inputs to this stage.
|
|
||||||
InputStages []int64
|
|
||||||
|
|
||||||
// Human-readable name for stage.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// ParallelInputs: Number of parallel input segments to be processed.
|
|
||||||
ParallelInputs int64
|
|
||||||
|
|
||||||
// ReadAvg: Duration the average shard spent reading input.
|
|
||||||
ReadAvg time.Duration
|
|
||||||
|
|
||||||
// ReadMax: Duration the slowest shard spent reading input.
|
|
||||||
ReadMax time.Duration
|
|
||||||
|
|
||||||
// Relative amount of the total time the average shard spent reading input.
|
|
||||||
ReadRatioAvg float64
|
|
||||||
|
|
||||||
// Relative amount of the total time the slowest shard spent reading input.
|
|
||||||
ReadRatioMax float64
|
|
||||||
|
|
||||||
// Number of records read into the stage.
|
|
||||||
RecordsRead int64
|
|
||||||
|
|
||||||
// Number of records written by the stage.
|
|
||||||
RecordsWritten int64
|
|
||||||
|
|
||||||
// ShuffleOutputBytes: Total number of bytes written to shuffle.
|
|
||||||
ShuffleOutputBytes int64
|
|
||||||
|
|
||||||
// ShuffleOutputBytesSpilled: Total number of bytes written to shuffle
|
|
||||||
// and spilled to disk.
|
|
||||||
ShuffleOutputBytesSpilled int64
|
|
||||||
|
|
||||||
// StartTime: Stage start time.
|
|
||||||
StartTime time.Time
|
|
||||||
|
|
||||||
// Current status for the stage.
|
|
||||||
Status string
|
|
||||||
|
|
||||||
// List of operations within the stage in dependency order (approximately
|
|
||||||
// chronological).
|
|
||||||
Steps []*ExplainQueryStep
|
|
||||||
|
|
||||||
// WaitAvg: Duration the average shard spent waiting to be scheduled.
|
|
||||||
WaitAvg time.Duration
|
|
||||||
|
|
||||||
// WaitMax: Duration the slowest shard spent waiting to be scheduled.
|
|
||||||
WaitMax time.Duration
|
|
||||||
|
|
||||||
// Relative amount of the total time the average shard spent waiting to be scheduled.
|
|
||||||
WaitRatioAvg float64
|
|
||||||
|
|
||||||
// Relative amount of the total time the slowest shard spent waiting to be scheduled.
|
|
||||||
WaitRatioMax float64
|
|
||||||
|
|
||||||
// WriteAvg: Duration the average shard spent on writing output.
|
|
||||||
WriteAvg time.Duration
|
|
||||||
|
|
||||||
// WriteMax: Duration the slowest shard spent on writing output.
|
|
||||||
WriteMax time.Duration
|
|
||||||
|
|
||||||
// Relative amount of the total time the average shard spent on writing output.
|
|
||||||
WriteRatioAvg float64
|
|
||||||
|
|
||||||
// Relative amount of the total time the slowest shard spent on writing output.
|
|
||||||
WriteRatioMax float64
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExplainQueryStep describes one step of a query stage.
|
|
||||||
type ExplainQueryStep struct {
|
|
||||||
// Machine-readable operation type.
|
|
||||||
Kind string
|
|
||||||
|
|
||||||
// Human-readable stage descriptions.
|
|
||||||
Substeps []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// QueryTimelineSample represents a sample of execution statistics at a point in time.
|
|
||||||
type QueryTimelineSample struct {
|
|
||||||
|
|
||||||
// Total number of units currently being processed by workers, represented as largest value since last sample.
|
|
||||||
ActiveUnits int64
|
|
||||||
|
|
||||||
// Total parallel units of work completed by this query.
|
|
||||||
CompletedUnits int64
|
|
||||||
|
|
||||||
// Time elapsed since start of query execution.
|
|
||||||
Elapsed time.Duration
|
|
||||||
|
|
||||||
// Total parallel units of work remaining for the active stages.
|
|
||||||
PendingUnits int64
|
|
||||||
|
|
||||||
// Cumulative slot-milliseconds consumed by the query.
|
|
||||||
SlotMillis int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*ExtractStatistics) implementsStatistics() {}
|
|
||||||
func (*LoadStatistics) implementsStatistics() {}
|
|
||||||
func (*QueryStatistics) implementsStatistics() {}
|
|
||||||
|
|
||||||
// Jobs lists jobs within a project.
|
|
||||||
func (c *Client) Jobs(ctx context.Context) *JobIterator {
|
|
||||||
it := &JobIterator{
|
|
||||||
ctx: ctx,
|
|
||||||
c: c,
|
|
||||||
ProjectID: c.projectID,
|
|
||||||
}
|
|
||||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
|
||||||
it.fetch,
|
|
||||||
func() int { return len(it.items) },
|
|
||||||
func() interface{} { b := it.items; it.items = nil; return b })
|
|
||||||
return it
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobIterator iterates over jobs in a project.
|
|
||||||
type JobIterator struct {
|
|
||||||
ProjectID string // Project ID of the jobs to list. Default is the client's project.
|
|
||||||
AllUsers bool // Whether to list jobs owned by all users in the project, or just the current caller.
|
|
||||||
State State // List only jobs in the given state. Defaults to all states.
|
|
||||||
|
|
||||||
ctx context.Context
|
|
||||||
c *Client
|
|
||||||
pageInfo *iterator.PageInfo
|
|
||||||
nextFunc func() error
|
|
||||||
items []*Job
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *JobIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
|
||||||
|
|
||||||
func (it *JobIterator) Next() (*Job, error) {
|
|
||||||
if err := it.nextFunc(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
item := it.items[0]
|
|
||||||
it.items = it.items[1:]
|
|
||||||
return item, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (it *JobIterator) fetch(pageSize int, pageToken string) (string, error) {
|
|
||||||
var st string
|
|
||||||
switch it.State {
|
|
||||||
case StateUnspecified:
|
|
||||||
st = ""
|
|
||||||
case Pending:
|
|
||||||
st = "pending"
|
|
||||||
case Running:
|
|
||||||
st = "running"
|
|
||||||
case Done:
|
|
||||||
st = "done"
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("bigquery: invalid value for JobIterator.State: %d", it.State)
|
|
||||||
}
|
|
||||||
|
|
||||||
req := it.c.bqs.Jobs.List(it.ProjectID).
|
|
||||||
Context(it.ctx).
|
|
||||||
PageToken(pageToken).
|
|
||||||
Projection("full").
|
|
||||||
AllUsers(it.AllUsers)
|
|
||||||
if st != "" {
|
|
||||||
req.StateFilter(st)
|
|
||||||
}
|
|
||||||
setClientHeader(req.Header())
|
|
||||||
if pageSize > 0 {
|
|
||||||
req.MaxResults(int64(pageSize))
|
|
||||||
}
|
|
||||||
res, err := req.Do()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, j := range res.Jobs {
|
|
||||||
job, err := convertListedJob(j, it.c)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
it.items = append(it.items, job)
|
|
||||||
}
|
|
||||||
return res.NextPageToken, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertListedJob(j *bq.JobListJobs, c *Client) (*Job, error) {
|
|
||||||
return bqToJob2(j.JobReference, j.Configuration, j.Status, j.Statistics, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Client) getJobInternal(ctx context.Context, jobID, location string, fields ...googleapi.Field) (*bq.Job, error) {
|
|
||||||
var job *bq.Job
|
|
||||||
call := c.bqs.Jobs.Get(c.projectID, jobID).Context(ctx)
|
|
||||||
if location != "" {
|
|
||||||
call = call.Location(location)
|
|
||||||
}
|
|
||||||
if len(fields) > 0 {
|
|
||||||
call = call.Fields(fields...)
|
|
||||||
}
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
err := runWithRetry(ctx, func() (err error) {
|
|
||||||
job, err = call.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return job, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToJob(q *bq.Job, c *Client) (*Job, error) {
|
|
||||||
return bqToJob2(q.JobReference, q.Configuration, q.Status, q.Statistics, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToJob2(qr *bq.JobReference, qc *bq.JobConfiguration, qs *bq.JobStatus, qt *bq.JobStatistics, c *Client) (*Job, error) {
|
|
||||||
j := &Job{
|
|
||||||
projectID: qr.ProjectId,
|
|
||||||
jobID: qr.JobId,
|
|
||||||
location: qr.Location,
|
|
||||||
c: c,
|
|
||||||
}
|
|
||||||
j.setConfig(qc)
|
|
||||||
if err := j.setStatus(qs); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
j.setStatistics(qt, c)
|
|
||||||
return j, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *Job) setConfig(config *bq.JobConfiguration) {
|
|
||||||
if config == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
j.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *Job) isQuery() bool {
|
|
||||||
return j.config != nil && j.config.Query != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done}
|
|
||||||
|
|
||||||
func (j *Job) setStatus(qs *bq.JobStatus) error {
|
|
||||||
if qs == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
state, ok := stateMap[qs.State]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("unexpected job state: %v", qs.State)
|
|
||||||
}
|
|
||||||
j.lastStatus = &JobStatus{
|
|
||||||
State: state,
|
|
||||||
err: nil,
|
|
||||||
}
|
|
||||||
if err := bqToError(qs.ErrorResult); state == Done && err != nil {
|
|
||||||
j.lastStatus.err = err
|
|
||||||
}
|
|
||||||
for _, ep := range qs.Errors {
|
|
||||||
j.lastStatus.Errors = append(j.lastStatus.Errors, bqToError(ep))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *Job) setStatistics(s *bq.JobStatistics, c *Client) {
|
|
||||||
if s == nil || j.lastStatus == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
js := &JobStatistics{
|
|
||||||
CreationTime: unixMillisToTime(s.CreationTime),
|
|
||||||
StartTime: unixMillisToTime(s.StartTime),
|
|
||||||
EndTime: unixMillisToTime(s.EndTime),
|
|
||||||
TotalBytesProcessed: s.TotalBytesProcessed,
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case s.Extract != nil:
|
|
||||||
js.Details = &ExtractStatistics{
|
|
||||||
DestinationURIFileCounts: []int64(s.Extract.DestinationUriFileCounts),
|
|
||||||
}
|
|
||||||
case s.Load != nil:
|
|
||||||
js.Details = &LoadStatistics{
|
|
||||||
InputFileBytes: s.Load.InputFileBytes,
|
|
||||||
InputFiles: s.Load.InputFiles,
|
|
||||||
OutputBytes: s.Load.OutputBytes,
|
|
||||||
OutputRows: s.Load.OutputRows,
|
|
||||||
}
|
|
||||||
case s.Query != nil:
|
|
||||||
var names []string
|
|
||||||
for _, qp := range s.Query.UndeclaredQueryParameters {
|
|
||||||
names = append(names, qp.Name)
|
|
||||||
}
|
|
||||||
var tables []*Table
|
|
||||||
for _, tr := range s.Query.ReferencedTables {
|
|
||||||
tables = append(tables, bqToTable(tr, c))
|
|
||||||
}
|
|
||||||
js.Details = &QueryStatistics{
|
|
||||||
BillingTier: s.Query.BillingTier,
|
|
||||||
CacheHit: s.Query.CacheHit,
|
|
||||||
DDLTargetTable: bqToTable(s.Query.DdlTargetTable, c),
|
|
||||||
DDLOperationPerformed: s.Query.DdlOperationPerformed,
|
|
||||||
StatementType: s.Query.StatementType,
|
|
||||||
TotalBytesBilled: s.Query.TotalBytesBilled,
|
|
||||||
TotalBytesProcessed: s.Query.TotalBytesProcessed,
|
|
||||||
NumDMLAffectedRows: s.Query.NumDmlAffectedRows,
|
|
||||||
QueryPlan: queryPlanFromProto(s.Query.QueryPlan),
|
|
||||||
Schema: bqToSchema(s.Query.Schema),
|
|
||||||
SlotMillis: s.Query.TotalSlotMs,
|
|
||||||
Timeline: timelineFromProto(s.Query.Timeline),
|
|
||||||
ReferencedTables: tables,
|
|
||||||
UndeclaredQueryParameterNames: names,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
j.lastStatus.Statistics = js
|
|
||||||
}
|
|
||||||
|
|
||||||
func queryPlanFromProto(stages []*bq.ExplainQueryStage) []*ExplainQueryStage {
|
|
||||||
var res []*ExplainQueryStage
|
|
||||||
for _, s := range stages {
|
|
||||||
var steps []*ExplainQueryStep
|
|
||||||
for _, p := range s.Steps {
|
|
||||||
steps = append(steps, &ExplainQueryStep{
|
|
||||||
Kind: p.Kind,
|
|
||||||
Substeps: p.Substeps,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
res = append(res, &ExplainQueryStage{
|
|
||||||
CompletedParallelInputs: s.CompletedParallelInputs,
|
|
||||||
ComputeAvg: time.Duration(s.ComputeMsAvg) * time.Millisecond,
|
|
||||||
ComputeMax: time.Duration(s.ComputeMsMax) * time.Millisecond,
|
|
||||||
ComputeRatioAvg: s.ComputeRatioAvg,
|
|
||||||
ComputeRatioMax: s.ComputeRatioMax,
|
|
||||||
EndTime: time.Unix(0, s.EndMs*1e6),
|
|
||||||
ID: s.Id,
|
|
||||||
InputStages: s.InputStages,
|
|
||||||
Name: s.Name,
|
|
||||||
ParallelInputs: s.ParallelInputs,
|
|
||||||
ReadAvg: time.Duration(s.ReadMsAvg) * time.Millisecond,
|
|
||||||
ReadMax: time.Duration(s.ReadMsMax) * time.Millisecond,
|
|
||||||
ReadRatioAvg: s.ReadRatioAvg,
|
|
||||||
ReadRatioMax: s.ReadRatioMax,
|
|
||||||
RecordsRead: s.RecordsRead,
|
|
||||||
RecordsWritten: s.RecordsWritten,
|
|
||||||
ShuffleOutputBytes: s.ShuffleOutputBytes,
|
|
||||||
ShuffleOutputBytesSpilled: s.ShuffleOutputBytesSpilled,
|
|
||||||
StartTime: time.Unix(0, s.StartMs*1e6),
|
|
||||||
Status: s.Status,
|
|
||||||
Steps: steps,
|
|
||||||
WaitAvg: time.Duration(s.WaitMsAvg) * time.Millisecond,
|
|
||||||
WaitMax: time.Duration(s.WaitMsMax) * time.Millisecond,
|
|
||||||
WaitRatioAvg: s.WaitRatioAvg,
|
|
||||||
WaitRatioMax: s.WaitRatioMax,
|
|
||||||
WriteAvg: time.Duration(s.WriteMsAvg) * time.Millisecond,
|
|
||||||
WriteMax: time.Duration(s.WriteMsMax) * time.Millisecond,
|
|
||||||
WriteRatioAvg: s.WriteRatioAvg,
|
|
||||||
WriteRatioMax: s.WriteRatioMax,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func timelineFromProto(timeline []*bq.QueryTimelineSample) []*QueryTimelineSample {
|
|
||||||
var res []*QueryTimelineSample
|
|
||||||
for _, s := range timeline {
|
|
||||||
res = append(res, &QueryTimelineSample{
|
|
||||||
ActiveUnits: s.ActiveUnits,
|
|
||||||
CompletedUnits: s.CompletedUnits,
|
|
||||||
Elapsed: time.Duration(s.ElapsedMs) * time.Millisecond,
|
|
||||||
PendingUnits: s.PendingUnits,
|
|
||||||
SlotMillis: s.TotalSlotMs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
95
vendor/cloud.google.com/go/bigquery/job_test.go
generated
vendored
95
vendor/cloud.google.com/go/bigquery/job_test.go
generated
vendored
@ -1,95 +0,0 @@
|
|||||||
// Copyright 2017 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCreateJobRef(t *testing.T) {
|
|
||||||
defer fixRandomID("RANDOM")()
|
|
||||||
cNoLoc := &Client{projectID: "projectID"}
|
|
||||||
cLoc := &Client{projectID: "projectID", Location: "defaultLoc"}
|
|
||||||
for _, test := range []struct {
|
|
||||||
in JobIDConfig
|
|
||||||
client *Client
|
|
||||||
want *bq.JobReference
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: JobIDConfig{JobID: "foo"},
|
|
||||||
want: &bq.JobReference{JobId: "foo"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{},
|
|
||||||
want: &bq.JobReference{JobId: "RANDOM"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{AddJobIDSuffix: true},
|
|
||||||
want: &bq.JobReference{JobId: "RANDOM"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{JobID: "foo", AddJobIDSuffix: true},
|
|
||||||
want: &bq.JobReference{JobId: "foo-RANDOM"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{JobID: "foo", Location: "loc"},
|
|
||||||
want: &bq.JobReference{JobId: "foo", Location: "loc"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{JobID: "foo"},
|
|
||||||
client: cLoc,
|
|
||||||
want: &bq.JobReference{JobId: "foo", Location: "defaultLoc"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: JobIDConfig{JobID: "foo", Location: "loc"},
|
|
||||||
client: cLoc,
|
|
||||||
want: &bq.JobReference{JobId: "foo", Location: "loc"},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
client := test.client
|
|
||||||
if client == nil {
|
|
||||||
client = cNoLoc
|
|
||||||
}
|
|
||||||
got := test.in.createJobRef(client)
|
|
||||||
test.want.ProjectId = "projectID"
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%+v: got %+v, want %+v", test.in, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fixRandomID(s string) func() {
|
|
||||||
prev := randomIDFn
|
|
||||||
randomIDFn = func() string { return s }
|
|
||||||
return func() { randomIDFn = prev }
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkJob(t *testing.T, i int, got, want *bq.Job) {
|
|
||||||
if got.JobReference == nil {
|
|
||||||
t.Errorf("#%d: empty job reference", i)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if got.JobReference.JobId == "" {
|
|
||||||
t.Errorf("#%d: empty job ID", i)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
d := testutil.Diff(got, want)
|
|
||||||
if d != "" {
|
|
||||||
t.Errorf("#%d: (got=-, want=+) %s", i, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
141
vendor/cloud.google.com/go/bigquery/load.go
generated
vendored
141
vendor/cloud.google.com/go/bigquery/load.go
generated
vendored
@ -1,141 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LoadConfig holds the configuration for a load job.
|
|
||||||
type LoadConfig struct {
|
|
||||||
// Src is the source from which data will be loaded.
|
|
||||||
Src LoadSource
|
|
||||||
|
|
||||||
// Dst is the table into which the data will be loaded.
|
|
||||||
Dst *Table
|
|
||||||
|
|
||||||
// CreateDisposition specifies the circumstances under which the destination table will be created.
|
|
||||||
// The default is CreateIfNeeded.
|
|
||||||
CreateDisposition TableCreateDisposition
|
|
||||||
|
|
||||||
// WriteDisposition specifies how existing data in the destination table is treated.
|
|
||||||
// The default is WriteAppend.
|
|
||||||
WriteDisposition TableWriteDisposition
|
|
||||||
|
|
||||||
// The labels associated with this job.
|
|
||||||
Labels map[string]string
|
|
||||||
|
|
||||||
// If non-nil, the destination table is partitioned by time.
|
|
||||||
TimePartitioning *TimePartitioning
|
|
||||||
|
|
||||||
// Custom encryption configuration (e.g., Cloud KMS keys).
|
|
||||||
DestinationEncryptionConfig *EncryptionConfig
|
|
||||||
|
|
||||||
// Allows the schema of the destination table to be updated as a side effect of
|
|
||||||
// the load job.
|
|
||||||
SchemaUpdateOptions []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *LoadConfig) toBQ() (*bq.JobConfiguration, io.Reader) {
|
|
||||||
config := &bq.JobConfiguration{
|
|
||||||
Labels: l.Labels,
|
|
||||||
Load: &bq.JobConfigurationLoad{
|
|
||||||
CreateDisposition: string(l.CreateDisposition),
|
|
||||||
WriteDisposition: string(l.WriteDisposition),
|
|
||||||
DestinationTable: l.Dst.toBQ(),
|
|
||||||
TimePartitioning: l.TimePartitioning.toBQ(),
|
|
||||||
DestinationEncryptionConfiguration: l.DestinationEncryptionConfig.toBQ(),
|
|
||||||
SchemaUpdateOptions: l.SchemaUpdateOptions,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
media := l.Src.populateLoadConfig(config.Load)
|
|
||||||
return config, media
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToLoadConfig(q *bq.JobConfiguration, c *Client) *LoadConfig {
|
|
||||||
lc := &LoadConfig{
|
|
||||||
Labels: q.Labels,
|
|
||||||
CreateDisposition: TableCreateDisposition(q.Load.CreateDisposition),
|
|
||||||
WriteDisposition: TableWriteDisposition(q.Load.WriteDisposition),
|
|
||||||
Dst: bqToTable(q.Load.DestinationTable, c),
|
|
||||||
TimePartitioning: bqToTimePartitioning(q.Load.TimePartitioning),
|
|
||||||
DestinationEncryptionConfig: bqToEncryptionConfig(q.Load.DestinationEncryptionConfiguration),
|
|
||||||
SchemaUpdateOptions: q.Load.SchemaUpdateOptions,
|
|
||||||
}
|
|
||||||
var fc *FileConfig
|
|
||||||
if len(q.Load.SourceUris) == 0 {
|
|
||||||
s := NewReaderSource(nil)
|
|
||||||
fc = &s.FileConfig
|
|
||||||
lc.Src = s
|
|
||||||
} else {
|
|
||||||
s := NewGCSReference(q.Load.SourceUris...)
|
|
||||||
fc = &s.FileConfig
|
|
||||||
lc.Src = s
|
|
||||||
}
|
|
||||||
bqPopulateFileConfig(q.Load, fc)
|
|
||||||
return lc
|
|
||||||
}
|
|
||||||
|
|
||||||
// A Loader loads data from Google Cloud Storage into a BigQuery table.
|
|
||||||
type Loader struct {
|
|
||||||
JobIDConfig
|
|
||||||
LoadConfig
|
|
||||||
c *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// A LoadSource represents a source of data that can be loaded into
|
|
||||||
// a BigQuery table.
|
|
||||||
//
|
|
||||||
// This package defines two LoadSources: GCSReference, for Google Cloud Storage
|
|
||||||
// objects, and ReaderSource, for data read from an io.Reader.
|
|
||||||
type LoadSource interface {
|
|
||||||
// populates config, returns media
|
|
||||||
populateLoadConfig(*bq.JobConfigurationLoad) io.Reader
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoaderFrom returns a Loader which can be used to load data into a BigQuery table.
|
|
||||||
// The returned Loader may optionally be further configured before its Run method is called.
|
|
||||||
// See GCSReference and ReaderSource for additional configuration options that
|
|
||||||
// affect loading.
|
|
||||||
func (t *Table) LoaderFrom(src LoadSource) *Loader {
|
|
||||||
return &Loader{
|
|
||||||
c: t.c,
|
|
||||||
LoadConfig: LoadConfig{
|
|
||||||
Src: src,
|
|
||||||
Dst: t,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run initiates a load job.
|
|
||||||
func (l *Loader) Run(ctx context.Context) (j *Job, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Load.Run")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
job, media := l.newJob()
|
|
||||||
return l.c.insertJob(ctx, job, media)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *Loader) newJob() (*bq.Job, io.Reader) {
|
|
||||||
config, media := l.LoadConfig.toBQ()
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: l.JobIDConfig.createJobRef(l.c),
|
|
||||||
Configuration: config,
|
|
||||||
}, media
|
|
||||||
}
|
|
||||||
260
vendor/cloud.google.com/go/bigquery/load_test.go
generated
vendored
260
vendor/cloud.google.com/go/bigquery/load_test.go
generated
vendored
@ -1,260 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
"github.com/google/go-cmp/cmp/cmpopts"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func defaultLoadJob() *bq.Job {
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
|
|
||||||
Configuration: &bq.JobConfiguration{
|
|
||||||
Load: &bq.JobConfigurationLoad{
|
|
||||||
DestinationTable: &bq.TableReference{
|
|
||||||
ProjectId: "client-project-id",
|
|
||||||
DatasetId: "dataset-id",
|
|
||||||
TableId: "table-id",
|
|
||||||
},
|
|
||||||
SourceUris: []string{"uri"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func stringFieldSchema() *FieldSchema {
|
|
||||||
return &FieldSchema{Name: "fieldname", Type: StringFieldType}
|
|
||||||
}
|
|
||||||
|
|
||||||
func nestedFieldSchema() *FieldSchema {
|
|
||||||
return &FieldSchema{
|
|
||||||
Name: "nested",
|
|
||||||
Type: RecordFieldType,
|
|
||||||
Schema: Schema{stringFieldSchema()},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqStringFieldSchema() *bq.TableFieldSchema {
|
|
||||||
return &bq.TableFieldSchema{
|
|
||||||
Name: "fieldname",
|
|
||||||
Type: "STRING",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqNestedFieldSchema() *bq.TableFieldSchema {
|
|
||||||
return &bq.TableFieldSchema{
|
|
||||||
Name: "nested",
|
|
||||||
Type: "RECORD",
|
|
||||||
Fields: []*bq.TableFieldSchema{bqStringFieldSchema()},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoad(t *testing.T) {
|
|
||||||
defer fixRandomID("RANDOM")()
|
|
||||||
c := &Client{projectID: "client-project-id"}
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
dst *Table
|
|
||||||
src LoadSource
|
|
||||||
jobID string
|
|
||||||
location string
|
|
||||||
config LoadConfig
|
|
||||||
want *bq.Job
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: NewGCSReference("uri"),
|
|
||||||
want: defaultLoadJob(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: NewGCSReference("uri"),
|
|
||||||
location: "loc",
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.JobReference.Location = "loc"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
jobID: "ajob",
|
|
||||||
config: LoadConfig{
|
|
||||||
CreateDisposition: CreateNever,
|
|
||||||
WriteDisposition: WriteTruncate,
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
TimePartitioning: &TimePartitioning{Expiration: 1234 * time.Millisecond},
|
|
||||||
DestinationEncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
||||||
SchemaUpdateOptions: []string{"ALLOW_FIELD_ADDITION"},
|
|
||||||
},
|
|
||||||
src: NewGCSReference("uri"),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.Configuration.Labels = map[string]string{"a": "b"}
|
|
||||||
j.Configuration.Load.CreateDisposition = "CREATE_NEVER"
|
|
||||||
j.Configuration.Load.WriteDisposition = "WRITE_TRUNCATE"
|
|
||||||
j.Configuration.Load.TimePartitioning = &bq.TimePartitioning{
|
|
||||||
Type: "DAY",
|
|
||||||
ExpirationMs: 1234,
|
|
||||||
}
|
|
||||||
j.Configuration.Load.DestinationEncryptionConfiguration = &bq.EncryptionConfiguration{KmsKeyName: "keyName"}
|
|
||||||
j.JobReference = &bq.JobReference{
|
|
||||||
JobId: "ajob",
|
|
||||||
ProjectId: "client-project-id",
|
|
||||||
}
|
|
||||||
j.Configuration.Load.SchemaUpdateOptions = []string{"ALLOW_FIELD_ADDITION"}
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.MaxBadRecords = 1
|
|
||||||
g.AllowJaggedRows = true
|
|
||||||
g.AllowQuotedNewlines = true
|
|
||||||
g.IgnoreUnknownValues = true
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.Configuration.Load.MaxBadRecords = 1
|
|
||||||
j.Configuration.Load.AllowJaggedRows = true
|
|
||||||
j.Configuration.Load.AllowQuotedNewlines = true
|
|
||||||
j.Configuration.Load.IgnoreUnknownValues = true
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.Schema = Schema{
|
|
||||||
stringFieldSchema(),
|
|
||||||
nestedFieldSchema(),
|
|
||||||
}
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.Configuration.Load.Schema = &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqStringFieldSchema(),
|
|
||||||
bqNestedFieldSchema(),
|
|
||||||
}}
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.SkipLeadingRows = 1
|
|
||||||
g.SourceFormat = JSON
|
|
||||||
g.Encoding = UTF_8
|
|
||||||
g.FieldDelimiter = "\t"
|
|
||||||
g.Quote = "-"
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.Configuration.Load.SkipLeadingRows = 1
|
|
||||||
j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON"
|
|
||||||
j.Configuration.Load.Encoding = "UTF-8"
|
|
||||||
j.Configuration.Load.FieldDelimiter = "\t"
|
|
||||||
hyphen := "-"
|
|
||||||
j.Configuration.Load.Quote = &hyphen
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: NewGCSReference("uri"),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
// Quote is left unset in GCSReference, so should be nil here.
|
|
||||||
j.Configuration.Load.Quote = nil
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.ForceZeroQuote = true
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
empty := ""
|
|
||||||
j.Configuration.Load.Quote = &empty
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: func() *ReaderSource {
|
|
||||||
r := NewReaderSource(strings.NewReader("foo"))
|
|
||||||
r.SkipLeadingRows = 1
|
|
||||||
r.SourceFormat = JSON
|
|
||||||
r.Encoding = UTF_8
|
|
||||||
r.FieldDelimiter = "\t"
|
|
||||||
r.Quote = "-"
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultLoadJob()
|
|
||||||
j.Configuration.Load.SourceUris = nil
|
|
||||||
j.Configuration.Load.SkipLeadingRows = 1
|
|
||||||
j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON"
|
|
||||||
j.Configuration.Load.Encoding = "UTF-8"
|
|
||||||
j.Configuration.Load.FieldDelimiter = "\t"
|
|
||||||
hyphen := "-"
|
|
||||||
j.Configuration.Load.Quote = &hyphen
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, tc := range testCases {
|
|
||||||
loader := tc.dst.LoaderFrom(tc.src)
|
|
||||||
loader.JobID = tc.jobID
|
|
||||||
loader.Location = tc.location
|
|
||||||
tc.config.Src = tc.src
|
|
||||||
tc.config.Dst = tc.dst
|
|
||||||
loader.LoadConfig = tc.config
|
|
||||||
got, _ := loader.newJob()
|
|
||||||
checkJob(t, i, got, tc.want)
|
|
||||||
|
|
||||||
jc, err := bqToJobConfig(got.Configuration, c)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("#%d: %v", i, err)
|
|
||||||
}
|
|
||||||
diff := testutil.Diff(jc.(*LoadConfig), &loader.LoadConfig,
|
|
||||||
cmp.AllowUnexported(Table{}, Client{}),
|
|
||||||
cmpopts.IgnoreUnexported(ReaderSource{}))
|
|
||||||
if diff != "" {
|
|
||||||
t.Errorf("#%d: (got=-, want=+:\n%s", i, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
299
vendor/cloud.google.com/go/bigquery/nulls.go
generated
vendored
299
vendor/cloud.google.com/go/bigquery/nulls.go
generated
vendored
@ -1,299 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NullInt64 represents a BigQuery INT64 that may be NULL.
|
|
||||||
type NullInt64 struct {
|
|
||||||
Int64 int64
|
|
||||||
Valid bool // Valid is true if Int64 is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullInt64) String() string { return nullstr(n.Valid, n.Int64) }
|
|
||||||
|
|
||||||
// NullString represents a BigQuery STRING that may be NULL.
|
|
||||||
type NullString struct {
|
|
||||||
StringVal string
|
|
||||||
Valid bool // Valid is true if StringVal is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullString) String() string { return nullstr(n.Valid, n.StringVal) }
|
|
||||||
|
|
||||||
// NullFloat64 represents a BigQuery FLOAT64 that may be NULL.
|
|
||||||
type NullFloat64 struct {
|
|
||||||
Float64 float64
|
|
||||||
Valid bool // Valid is true if Float64 is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullFloat64) String() string { return nullstr(n.Valid, n.Float64) }
|
|
||||||
|
|
||||||
// NullBool represents a BigQuery BOOL that may be NULL.
|
|
||||||
type NullBool struct {
|
|
||||||
Bool bool
|
|
||||||
Valid bool // Valid is true if Bool is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullBool) String() string { return nullstr(n.Valid, n.Bool) }
|
|
||||||
|
|
||||||
// NullTimestamp represents a BigQuery TIMESTAMP that may be null.
|
|
||||||
type NullTimestamp struct {
|
|
||||||
Timestamp time.Time
|
|
||||||
Valid bool // Valid is true if Time is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullTimestamp) String() string { return nullstr(n.Valid, n.Timestamp) }
|
|
||||||
|
|
||||||
// NullDate represents a BigQuery DATE that may be null.
|
|
||||||
type NullDate struct {
|
|
||||||
Date civil.Date
|
|
||||||
Valid bool // Valid is true if Date is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullDate) String() string { return nullstr(n.Valid, n.Date) }
|
|
||||||
|
|
||||||
// NullTime represents a BigQuery TIME that may be null.
|
|
||||||
type NullTime struct {
|
|
||||||
Time civil.Time
|
|
||||||
Valid bool // Valid is true if Time is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullTime) String() string {
|
|
||||||
if !n.Valid {
|
|
||||||
return "<null>"
|
|
||||||
}
|
|
||||||
return CivilTimeString(n.Time)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NullDateTime represents a BigQuery DATETIME that may be null.
|
|
||||||
type NullDateTime struct {
|
|
||||||
DateTime civil.DateTime
|
|
||||||
Valid bool // Valid is true if DateTime is not NULL.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullDateTime) String() string {
|
|
||||||
if !n.Valid {
|
|
||||||
return "<null>"
|
|
||||||
}
|
|
||||||
return CivilDateTimeString(n.DateTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullInt64) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.Int64) }
|
|
||||||
func (n NullFloat64) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.Float64) }
|
|
||||||
func (n NullBool) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.Bool) }
|
|
||||||
func (n NullString) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.StringVal) }
|
|
||||||
func (n NullTimestamp) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.Timestamp) }
|
|
||||||
func (n NullDate) MarshalJSON() ([]byte, error) { return nulljson(n.Valid, n.Date) }
|
|
||||||
|
|
||||||
func (n NullTime) MarshalJSON() ([]byte, error) {
|
|
||||||
if !n.Valid {
|
|
||||||
return jsonNull, nil
|
|
||||||
}
|
|
||||||
return []byte(`"` + CivilTimeString(n.Time) + `"`), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n NullDateTime) MarshalJSON() ([]byte, error) {
|
|
||||||
if !n.Valid {
|
|
||||||
return jsonNull, nil
|
|
||||||
}
|
|
||||||
return []byte(`"` + CivilDateTimeString(n.DateTime) + `"`), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func nullstr(valid bool, v interface{}) string {
|
|
||||||
if !valid {
|
|
||||||
return "NULL"
|
|
||||||
}
|
|
||||||
return fmt.Sprint(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
var jsonNull = []byte("null")
|
|
||||||
|
|
||||||
func nulljson(valid bool, v interface{}) ([]byte, error) {
|
|
||||||
if !valid {
|
|
||||||
return jsonNull, nil
|
|
||||||
}
|
|
||||||
return json.Marshal(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullInt64) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Int64 = 0
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.Int64); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullFloat64) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Float64 = 0
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.Float64); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullBool) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Bool = false
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.Bool); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullString) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.StringVal = ""
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.StringVal); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullTimestamp) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Timestamp = time.Time{}
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.Timestamp); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullDate) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Date = civil.Date{}
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal(b, &n.Date); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullTime) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.Time = civil.Time{}
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
s, err := strconv.Unquote(string(b))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
t, err := civil.ParseTime(s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.Time = t
|
|
||||||
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *NullDateTime) UnmarshalJSON(b []byte) error {
|
|
||||||
n.Valid = false
|
|
||||||
n.DateTime = civil.DateTime{}
|
|
||||||
if bytes.Equal(b, jsonNull) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
s, err := strconv.Unquote(string(b))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dt, err := parseCivilDateTime(s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n.DateTime = dt
|
|
||||||
|
|
||||||
n.Valid = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
typeOfNullInt64 = reflect.TypeOf(NullInt64{})
|
|
||||||
typeOfNullFloat64 = reflect.TypeOf(NullFloat64{})
|
|
||||||
typeOfNullBool = reflect.TypeOf(NullBool{})
|
|
||||||
typeOfNullString = reflect.TypeOf(NullString{})
|
|
||||||
typeOfNullTimestamp = reflect.TypeOf(NullTimestamp{})
|
|
||||||
typeOfNullDate = reflect.TypeOf(NullDate{})
|
|
||||||
typeOfNullTime = reflect.TypeOf(NullTime{})
|
|
||||||
typeOfNullDateTime = reflect.TypeOf(NullDateTime{})
|
|
||||||
)
|
|
||||||
|
|
||||||
func nullableFieldType(t reflect.Type) FieldType {
|
|
||||||
switch t {
|
|
||||||
case typeOfNullInt64:
|
|
||||||
return IntegerFieldType
|
|
||||||
case typeOfNullFloat64:
|
|
||||||
return FloatFieldType
|
|
||||||
case typeOfNullBool:
|
|
||||||
return BooleanFieldType
|
|
||||||
case typeOfNullString:
|
|
||||||
return StringFieldType
|
|
||||||
case typeOfNullTimestamp:
|
|
||||||
return TimestampFieldType
|
|
||||||
case typeOfNullDate:
|
|
||||||
return DateFieldType
|
|
||||||
case typeOfNullTime:
|
|
||||||
return TimeFieldType
|
|
||||||
case typeOfNullDateTime:
|
|
||||||
return DateTimeFieldType
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
73
vendor/cloud.google.com/go/bigquery/nulls_test.go
generated
vendored
73
vendor/cloud.google.com/go/bigquery/nulls_test.go
generated
vendored
@ -1,73 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
nullsTestTime = civil.Time{Hour: 7, Minute: 50, Second: 22, Nanosecond: 1000}
|
|
||||||
nullsTestDateTime = civil.DateTime{Date: civil.Date{Year: 2016, Month: 11, Day: 5}, Time: nullsTestTime}
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNullsJSON(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
in interface{}
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{&NullInt64{Valid: true, Int64: 3}, `3`},
|
|
||||||
{&NullFloat64{Valid: true, Float64: 3.14}, `3.14`},
|
|
||||||
{&NullBool{Valid: true, Bool: true}, `true`},
|
|
||||||
{&NullString{Valid: true, StringVal: "foo"}, `"foo"`},
|
|
||||||
{&NullTimestamp{Valid: true, Timestamp: testTimestamp}, `"2016-11-05T07:50:22.000000008Z"`},
|
|
||||||
{&NullDate{Valid: true, Date: testDate}, `"2016-11-05"`},
|
|
||||||
{&NullTime{Valid: true, Time: nullsTestTime}, `"07:50:22.000001"`},
|
|
||||||
{&NullDateTime{Valid: true, DateTime: nullsTestDateTime}, `"2016-11-05 07:50:22.000001"`},
|
|
||||||
|
|
||||||
{&NullInt64{}, `null`},
|
|
||||||
{&NullFloat64{}, `null`},
|
|
||||||
{&NullBool{}, `null`},
|
|
||||||
{&NullString{}, `null`},
|
|
||||||
{&NullTimestamp{}, `null`},
|
|
||||||
{&NullDate{}, `null`},
|
|
||||||
{&NullTime{}, `null`},
|
|
||||||
{&NullDateTime{}, `null`},
|
|
||||||
} {
|
|
||||||
bytes, err := json.Marshal(test.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, want := string(bytes), test.want; got != want {
|
|
||||||
t.Errorf("%#v: got %s, want %s", test.in, got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
typ := reflect.Indirect(reflect.ValueOf(test.in)).Type()
|
|
||||||
value := reflect.New(typ).Interface()
|
|
||||||
err = json.Unmarshal(bytes, value)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !testutil.Equal(value, test.in) {
|
|
||||||
t.Errorf("%#v: got %#v, want %#v", test.in, value, test.in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
40
vendor/cloud.google.com/go/bigquery/oc_test.go
generated
vendored
40
vendor/cloud.google.com/go/bigquery/oc_test.go
generated
vendored
@ -1,40 +0,0 @@
|
|||||||
// Copyright 2018 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// +build go1.8
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOCTracing(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
client := getClient(t)
|
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
te := testutil.NewTestExporter()
|
|
||||||
defer te.Unregister()
|
|
||||||
|
|
||||||
q := client.Query("select *")
|
|
||||||
q.Run(ctx) // Doesn't matter if we get an error; span should be created either way
|
|
||||||
|
|
||||||
if len(te.Spans) == 0 {
|
|
||||||
t.Fatalf("Expected some spans to be created, but got %d", 0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
357
vendor/cloud.google.com/go/bigquery/params.go
generated
vendored
357
vendor/cloud.google.com/go/bigquery/params.go
generated
vendored
@ -1,357 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math/big"
|
|
||||||
"reflect"
|
|
||||||
"regexp"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
"cloud.google.com/go/internal/fields"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
// See https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#timestamp-type.
|
|
||||||
timestampFormat = "2006-01-02 15:04:05.999999-07:00"
|
|
||||||
|
|
||||||
// See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#schema.fields.name
|
|
||||||
validFieldName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]{0,127}$")
|
|
||||||
)
|
|
||||||
|
|
||||||
const nullableTagOption = "nullable"
|
|
||||||
|
|
||||||
func bqTagParser(t reflect.StructTag) (name string, keep bool, other interface{}, err error) {
|
|
||||||
name, keep, opts, err := fields.ParseStandardTag("bigquery", t)
|
|
||||||
if err != nil {
|
|
||||||
return "", false, nil, err
|
|
||||||
}
|
|
||||||
if name != "" && !validFieldName.MatchString(name) {
|
|
||||||
return "", false, nil, errInvalidFieldName
|
|
||||||
}
|
|
||||||
for _, opt := range opts {
|
|
||||||
if opt != nullableTagOption {
|
|
||||||
return "", false, nil, fmt.Errorf(
|
|
||||||
"bigquery: invalid tag option %q. The only valid option is %q",
|
|
||||||
opt, nullableTagOption)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return name, keep, opts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var fieldCache = fields.NewCache(bqTagParser, nil, nil)
|
|
||||||
|
|
||||||
var (
|
|
||||||
int64ParamType = &bq.QueryParameterType{Type: "INT64"}
|
|
||||||
float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
|
|
||||||
boolParamType = &bq.QueryParameterType{Type: "BOOL"}
|
|
||||||
stringParamType = &bq.QueryParameterType{Type: "STRING"}
|
|
||||||
bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
|
|
||||||
dateParamType = &bq.QueryParameterType{Type: "DATE"}
|
|
||||||
timeParamType = &bq.QueryParameterType{Type: "TIME"}
|
|
||||||
dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
|
|
||||||
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
|
|
||||||
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
typeOfDate = reflect.TypeOf(civil.Date{})
|
|
||||||
typeOfTime = reflect.TypeOf(civil.Time{})
|
|
||||||
typeOfDateTime = reflect.TypeOf(civil.DateTime{})
|
|
||||||
typeOfGoTime = reflect.TypeOf(time.Time{})
|
|
||||||
typeOfRat = reflect.TypeOf(&big.Rat{})
|
|
||||||
)
|
|
||||||
|
|
||||||
// A QueryParameter is a parameter to a query.
|
|
||||||
type QueryParameter struct {
|
|
||||||
// Name is used for named parameter mode.
|
|
||||||
// It must match the name in the query case-insensitively.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// Value is the value of the parameter.
|
|
||||||
//
|
|
||||||
// When you create a QueryParameter to send to BigQuery, the following Go types
|
|
||||||
// are supported, with their corresponding Bigquery types:
|
|
||||||
// int, int8, int16, int32, int64, uint8, uint16, uint32: INT64
|
|
||||||
// Note that uint, uint64 and uintptr are not supported, because
|
|
||||||
// they may contain values that cannot fit into a 64-bit signed integer.
|
|
||||||
// float32, float64: FLOAT64
|
|
||||||
// bool: BOOL
|
|
||||||
// string: STRING
|
|
||||||
// []byte: BYTES
|
|
||||||
// time.Time: TIMESTAMP
|
|
||||||
// *big.Rat: NUMERIC
|
|
||||||
// Arrays and slices of the above.
|
|
||||||
// Structs of the above. Only the exported fields are used.
|
|
||||||
//
|
|
||||||
// When a QueryParameter is returned inside a QueryConfig from a call to
|
|
||||||
// Job.Config:
|
|
||||||
// Integers are of type int64.
|
|
||||||
// Floating-point values are of type float64.
|
|
||||||
// Arrays are of type []interface{}, regardless of the array element type.
|
|
||||||
// Structs are of type map[string]interface{}.
|
|
||||||
Value interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
|
|
||||||
pv, err := paramValue(reflect.ValueOf(p.Value))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pt, err := paramType(reflect.TypeOf(p.Value))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &bq.QueryParameter{
|
|
||||||
Name: p.Name,
|
|
||||||
ParameterValue: &pv,
|
|
||||||
ParameterType: pt,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
|
|
||||||
if t == nil {
|
|
||||||
return nil, errors.New("bigquery: nil parameter")
|
|
||||||
}
|
|
||||||
switch t {
|
|
||||||
case typeOfDate:
|
|
||||||
return dateParamType, nil
|
|
||||||
case typeOfTime:
|
|
||||||
return timeParamType, nil
|
|
||||||
case typeOfDateTime:
|
|
||||||
return dateTimeParamType, nil
|
|
||||||
case typeOfGoTime:
|
|
||||||
return timestampParamType, nil
|
|
||||||
case typeOfRat:
|
|
||||||
return numericParamType, nil
|
|
||||||
}
|
|
||||||
switch t.Kind() {
|
|
||||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32:
|
|
||||||
return int64ParamType, nil
|
|
||||||
|
|
||||||
case reflect.Float32, reflect.Float64:
|
|
||||||
return float64ParamType, nil
|
|
||||||
|
|
||||||
case reflect.Bool:
|
|
||||||
return boolParamType, nil
|
|
||||||
|
|
||||||
case reflect.String:
|
|
||||||
return stringParamType, nil
|
|
||||||
|
|
||||||
case reflect.Slice:
|
|
||||||
if t.Elem().Kind() == reflect.Uint8 {
|
|
||||||
return bytesParamType, nil
|
|
||||||
}
|
|
||||||
fallthrough
|
|
||||||
|
|
||||||
case reflect.Array:
|
|
||||||
et, err := paramType(t.Elem())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &bq.QueryParameterType{Type: "ARRAY", ArrayType: et}, nil
|
|
||||||
|
|
||||||
case reflect.Ptr:
|
|
||||||
if t.Elem().Kind() != reflect.Struct {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
t = t.Elem()
|
|
||||||
fallthrough
|
|
||||||
|
|
||||||
case reflect.Struct:
|
|
||||||
var fts []*bq.QueryParameterTypeStructTypes
|
|
||||||
fields, err := fieldCache.Fields(t)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, f := range fields {
|
|
||||||
pt, err := paramType(f.Type)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fts = append(fts, &bq.QueryParameterTypeStructTypes{
|
|
||||||
Name: f.Name,
|
|
||||||
Type: pt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return &bq.QueryParameterType{Type: "STRUCT", StructTypes: fts}, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter type", t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
|
|
||||||
var res bq.QueryParameterValue
|
|
||||||
if !v.IsValid() {
|
|
||||||
return res, errors.New("bigquery: nil parameter")
|
|
||||||
}
|
|
||||||
t := v.Type()
|
|
||||||
switch t {
|
|
||||||
case typeOfDate:
|
|
||||||
res.Value = v.Interface().(civil.Date).String()
|
|
||||||
return res, nil
|
|
||||||
|
|
||||||
case typeOfTime:
|
|
||||||
// civil.Time has nanosecond resolution, but BigQuery TIME only microsecond.
|
|
||||||
// (If we send nanoseconds, then when we try to read the result we get "query job
|
|
||||||
// missing destination table").
|
|
||||||
res.Value = CivilTimeString(v.Interface().(civil.Time))
|
|
||||||
return res, nil
|
|
||||||
|
|
||||||
case typeOfDateTime:
|
|
||||||
res.Value = CivilDateTimeString(v.Interface().(civil.DateTime))
|
|
||||||
return res, nil
|
|
||||||
|
|
||||||
case typeOfGoTime:
|
|
||||||
res.Value = v.Interface().(time.Time).Format(timestampFormat)
|
|
||||||
return res, nil
|
|
||||||
|
|
||||||
case typeOfRat:
|
|
||||||
res.Value = NumericString(v.Interface().(*big.Rat))
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
switch t.Kind() {
|
|
||||||
case reflect.Slice:
|
|
||||||
if t.Elem().Kind() == reflect.Uint8 {
|
|
||||||
res.Value = base64.StdEncoding.EncodeToString(v.Interface().([]byte))
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
fallthrough
|
|
||||||
|
|
||||||
case reflect.Array:
|
|
||||||
var vals []*bq.QueryParameterValue
|
|
||||||
for i := 0; i < v.Len(); i++ {
|
|
||||||
val, err := paramValue(v.Index(i))
|
|
||||||
if err != nil {
|
|
||||||
return bq.QueryParameterValue{}, err
|
|
||||||
}
|
|
||||||
vals = append(vals, &val)
|
|
||||||
}
|
|
||||||
return bq.QueryParameterValue{ArrayValues: vals}, nil
|
|
||||||
|
|
||||||
case reflect.Ptr:
|
|
||||||
if t.Elem().Kind() != reflect.Struct {
|
|
||||||
return res, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter value", t)
|
|
||||||
}
|
|
||||||
t = t.Elem()
|
|
||||||
v = v.Elem()
|
|
||||||
if !v.IsValid() {
|
|
||||||
// nil pointer becomes empty value
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
fallthrough
|
|
||||||
|
|
||||||
case reflect.Struct:
|
|
||||||
fields, err := fieldCache.Fields(t)
|
|
||||||
if err != nil {
|
|
||||||
return bq.QueryParameterValue{}, err
|
|
||||||
}
|
|
||||||
res.StructValues = map[string]bq.QueryParameterValue{}
|
|
||||||
for _, f := range fields {
|
|
||||||
fv := v.FieldByIndex(f.Index)
|
|
||||||
fp, err := paramValue(fv)
|
|
||||||
if err != nil {
|
|
||||||
return bq.QueryParameterValue{}, err
|
|
||||||
}
|
|
||||||
res.StructValues[f.Name] = fp
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
// None of the above: assume a scalar type. (If it's not a valid type,
|
|
||||||
// paramType will catch the error.)
|
|
||||||
res.Value = fmt.Sprint(v.Interface())
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToQueryParameter(q *bq.QueryParameter) (QueryParameter, error) {
|
|
||||||
p := QueryParameter{Name: q.Name}
|
|
||||||
val, err := convertParamValue(q.ParameterValue, q.ParameterType)
|
|
||||||
if err != nil {
|
|
||||||
return QueryParameter{}, err
|
|
||||||
}
|
|
||||||
p.Value = val
|
|
||||||
return p, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var paramTypeToFieldType = map[string]FieldType{
|
|
||||||
int64ParamType.Type: IntegerFieldType,
|
|
||||||
float64ParamType.Type: FloatFieldType,
|
|
||||||
boolParamType.Type: BooleanFieldType,
|
|
||||||
stringParamType.Type: StringFieldType,
|
|
||||||
bytesParamType.Type: BytesFieldType,
|
|
||||||
dateParamType.Type: DateFieldType,
|
|
||||||
timeParamType.Type: TimeFieldType,
|
|
||||||
numericParamType.Type: NumericFieldType,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert a parameter value from the service to a Go value. This is similar to, but
|
|
||||||
// not quite the same as, converting data values.
|
|
||||||
func convertParamValue(qval *bq.QueryParameterValue, qtype *bq.QueryParameterType) (interface{}, error) {
|
|
||||||
switch qtype.Type {
|
|
||||||
case "ARRAY":
|
|
||||||
if qval == nil {
|
|
||||||
return []interface{}(nil), nil
|
|
||||||
}
|
|
||||||
return convertParamArray(qval.ArrayValues, qtype.ArrayType)
|
|
||||||
case "STRUCT":
|
|
||||||
if qval == nil {
|
|
||||||
return map[string]interface{}(nil), nil
|
|
||||||
}
|
|
||||||
return convertParamStruct(qval.StructValues, qtype.StructTypes)
|
|
||||||
case "TIMESTAMP":
|
|
||||||
return time.Parse(timestampFormat, qval.Value)
|
|
||||||
case "DATETIME":
|
|
||||||
return parseCivilDateTime(qval.Value)
|
|
||||||
default:
|
|
||||||
return convertBasicType(qval.Value, paramTypeToFieldType[qtype.Type])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// convertParamArray converts a query parameter array value to a Go value. It
|
|
||||||
// always returns a []interface{}.
|
|
||||||
func convertParamArray(elVals []*bq.QueryParameterValue, elType *bq.QueryParameterType) ([]interface{}, error) {
|
|
||||||
var vals []interface{}
|
|
||||||
for _, el := range elVals {
|
|
||||||
val, err := convertParamValue(el, elType)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
vals = append(vals, val)
|
|
||||||
}
|
|
||||||
return vals, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// convertParamStruct converts a query parameter struct value into a Go value. It
|
|
||||||
// always returns a map[string]interface{}.
|
|
||||||
func convertParamStruct(sVals map[string]bq.QueryParameterValue, sTypes []*bq.QueryParameterTypeStructTypes) (map[string]interface{}, error) {
|
|
||||||
vals := map[string]interface{}{}
|
|
||||||
for _, st := range sTypes {
|
|
||||||
if sv, ok := sVals[st.Name]; ok {
|
|
||||||
val, err := convertParamValue(&sv, st.Type)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
vals[st.Name] = val
|
|
||||||
} else {
|
|
||||||
vals[st.Name] = nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return vals, nil
|
|
||||||
}
|
|
||||||
363
vendor/cloud.google.com/go/bigquery/params_test.go
generated
vendored
363
vendor/cloud.google.com/go/bigquery/params_test.go
generated
vendored
@ -1,363 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"math"
|
|
||||||
"math/big"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
var scalarTests = []struct {
|
|
||||||
val interface{} // The Go value
|
|
||||||
wantVal string // paramValue's desired output
|
|
||||||
wantType *bq.QueryParameterType // paramType's desired output
|
|
||||||
}{
|
|
||||||
{int64(0), "0", int64ParamType},
|
|
||||||
{3.14, "3.14", float64ParamType},
|
|
||||||
{3.14159e-87, "3.14159e-87", float64ParamType},
|
|
||||||
{true, "true", boolParamType},
|
|
||||||
{"string", "string", stringParamType},
|
|
||||||
{"\u65e5\u672c\u8a9e\n", "\u65e5\u672c\u8a9e\n", stringParamType},
|
|
||||||
{math.NaN(), "NaN", float64ParamType},
|
|
||||||
{[]byte("foo"), "Zm9v", bytesParamType}, // base64 encoding of "foo"
|
|
||||||
{time.Date(2016, 3, 20, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)),
|
|
||||||
"2016-03-20 04:22:09.000005-01:02",
|
|
||||||
timestampParamType},
|
|
||||||
{civil.Date{Year: 2016, Month: 3, Day: 20}, "2016-03-20", dateParamType},
|
|
||||||
{civil.Time{Hour: 4, Minute: 5, Second: 6, Nanosecond: 789000000}, "04:05:06.789000", timeParamType},
|
|
||||||
{civil.DateTime{Date: civil.Date{Year: 2016, Month: 3, Day: 20}, Time: civil.Time{Hour: 4, Minute: 5, Second: 6, Nanosecond: 789000000}},
|
|
||||||
"2016-03-20 04:05:06.789000",
|
|
||||||
dateTimeParamType},
|
|
||||||
{big.NewRat(12345, 1000), "12.345000000", numericParamType},
|
|
||||||
}
|
|
||||||
|
|
||||||
type (
|
|
||||||
S1 struct {
|
|
||||||
A int
|
|
||||||
B *S2
|
|
||||||
C bool
|
|
||||||
}
|
|
||||||
S2 struct {
|
|
||||||
D string
|
|
||||||
e int
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
s1 = S1{
|
|
||||||
A: 1,
|
|
||||||
B: &S2{D: "s"},
|
|
||||||
C: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
s1ParamType = &bq.QueryParameterType{
|
|
||||||
Type: "STRUCT",
|
|
||||||
StructTypes: []*bq.QueryParameterTypeStructTypes{
|
|
||||||
{Name: "A", Type: int64ParamType},
|
|
||||||
{Name: "B", Type: &bq.QueryParameterType{
|
|
||||||
Type: "STRUCT",
|
|
||||||
StructTypes: []*bq.QueryParameterTypeStructTypes{
|
|
||||||
{Name: "D", Type: stringParamType},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
{Name: "C", Type: boolParamType},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
s1ParamValue = bq.QueryParameterValue{
|
|
||||||
StructValues: map[string]bq.QueryParameterValue{
|
|
||||||
"A": sval("1"),
|
|
||||||
"B": {
|
|
||||||
StructValues: map[string]bq.QueryParameterValue{
|
|
||||||
"D": sval("s"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"C": sval("true"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
s1ParamReturnValue = map[string]interface{}{
|
|
||||||
"A": int64(1),
|
|
||||||
"B": map[string]interface{}{"D": "s"},
|
|
||||||
"C": true,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func sval(s string) bq.QueryParameterValue {
|
|
||||||
return bq.QueryParameterValue{Value: s}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamValueScalar(t *testing.T) {
|
|
||||||
for _, test := range scalarTests {
|
|
||||||
got, err := paramValue(reflect.ValueOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%v: got %v, want nil", test.val, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
want := sval(test.wantVal)
|
|
||||||
if !testutil.Equal(got, want) {
|
|
||||||
t.Errorf("%v:\ngot %+v\nwant %+v", test.val, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamValueArray(t *testing.T) {
|
|
||||||
qpv := bq.QueryParameterValue{ArrayValues: []*bq.QueryParameterValue{
|
|
||||||
{Value: "1"},
|
|
||||||
{Value: "2"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
val interface{}
|
|
||||||
want bq.QueryParameterValue
|
|
||||||
}{
|
|
||||||
{[]int(nil), bq.QueryParameterValue{}},
|
|
||||||
{[]int{}, bq.QueryParameterValue{}},
|
|
||||||
{[]int{1, 2}, qpv},
|
|
||||||
{[2]int{1, 2}, qpv},
|
|
||||||
} {
|
|
||||||
got, err := paramValue(reflect.ValueOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%#v:\ngot %+v\nwant %+v", test.val, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamValueStruct(t *testing.T) {
|
|
||||||
got, err := paramValue(reflect.ValueOf(s1))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, s1ParamValue) {
|
|
||||||
t.Errorf("got %+v\nwant %+v", got, s1ParamValue)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamValueErrors(t *testing.T) {
|
|
||||||
// paramValue lets a few invalid types through, but paramType catches them.
|
|
||||||
// Since we never call one without the other that's fine.
|
|
||||||
for _, val := range []interface{}{nil, new([]int)} {
|
|
||||||
_, err := paramValue(reflect.ValueOf(val))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%v (%T): got nil, want error", val, val)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamType(t *testing.T) {
|
|
||||||
for _, test := range scalarTests {
|
|
||||||
got, err := paramType(reflect.TypeOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.wantType) {
|
|
||||||
t.Errorf("%v (%T): got %v, want %v", test.val, test.val, got, test.wantType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
val interface{}
|
|
||||||
want *bq.QueryParameterType
|
|
||||||
}{
|
|
||||||
{uint32(32767), int64ParamType},
|
|
||||||
{[]byte("foo"), bytesParamType},
|
|
||||||
{[]int{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: int64ParamType}},
|
|
||||||
{[3]bool{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: boolParamType}},
|
|
||||||
{S1{}, s1ParamType},
|
|
||||||
} {
|
|
||||||
got, err := paramType(reflect.TypeOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%v (%T): got %v, want %v", test.val, test.val, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParamTypeErrors(t *testing.T) {
|
|
||||||
for _, val := range []interface{}{
|
|
||||||
nil, uint(0), new([]int), make(chan int),
|
|
||||||
} {
|
|
||||||
_, err := paramType(reflect.TypeOf(val))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%v (%T): got nil, want error", val, val)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConvertParamValue(t *testing.T) {
|
|
||||||
// Scalars.
|
|
||||||
for _, test := range scalarTests {
|
|
||||||
pval, err := paramValue(reflect.ValueOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ptype, err := paramType(reflect.TypeOf(test.val))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
got, err := convertParamValue(&pval, ptype)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("convertParamValue(%+v, %+v): %v", pval, ptype, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.val) {
|
|
||||||
t.Errorf("%#v: got %#v", test.val, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Arrays.
|
|
||||||
for _, test := range []struct {
|
|
||||||
pval *bq.QueryParameterValue
|
|
||||||
want []interface{}
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
&bq.QueryParameterValue{},
|
|
||||||
nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
&bq.QueryParameterValue{
|
|
||||||
ArrayValues: []*bq.QueryParameterValue{{Value: "1"}, {Value: "2"}},
|
|
||||||
},
|
|
||||||
[]interface{}{int64(1), int64(2)},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
ptype := &bq.QueryParameterType{Type: "ARRAY", ArrayType: int64ParamType}
|
|
||||||
got, err := convertParamValue(test.pval, ptype)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v: %v", test.pval, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%+v: got %+v, want %+v", test.pval, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Structs.
|
|
||||||
got, err := convertParamValue(&s1ParamValue, s1ParamType)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, s1ParamReturnValue) {
|
|
||||||
t.Errorf("got %+v, want %+v", got, s1ParamReturnValue)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegration_ScalarParam(t *testing.T) {
|
|
||||||
roundToMicros := cmp.Transformer("RoundToMicros",
|
|
||||||
func(t time.Time) time.Time { return t.Round(time.Microsecond) })
|
|
||||||
c := getClient(t)
|
|
||||||
for _, test := range scalarTests {
|
|
||||||
gotData, gotParam, err := paramRoundTrip(c, test.val)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(gotData, test.val, roundToMicros) {
|
|
||||||
t.Errorf("\ngot %#v (%T)\nwant %#v (%T)", gotData, gotData, test.val, test.val)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(gotParam, test.val, roundToMicros) {
|
|
||||||
t.Errorf("\ngot %#v (%T)\nwant %#v (%T)", gotParam, gotParam, test.val, test.val)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntegration_OtherParam(t *testing.T) {
|
|
||||||
c := getClient(t)
|
|
||||||
for _, test := range []struct {
|
|
||||||
val interface{}
|
|
||||||
wantData interface{}
|
|
||||||
wantParam interface{}
|
|
||||||
}{
|
|
||||||
{[]int(nil), []Value(nil), []interface{}(nil)},
|
|
||||||
{[]int{}, []Value(nil), []interface{}(nil)},
|
|
||||||
{
|
|
||||||
[]int{1, 2},
|
|
||||||
[]Value{int64(1), int64(2)},
|
|
||||||
[]interface{}{int64(1), int64(2)},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[3]int{1, 2, 3},
|
|
||||||
[]Value{int64(1), int64(2), int64(3)},
|
|
||||||
[]interface{}{int64(1), int64(2), int64(3)},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
S1{},
|
|
||||||
[]Value{int64(0), nil, false},
|
|
||||||
map[string]interface{}{
|
|
||||||
"A": int64(0),
|
|
||||||
"B": nil,
|
|
||||||
"C": false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
s1,
|
|
||||||
[]Value{int64(1), []Value{"s"}, true},
|
|
||||||
s1ParamReturnValue,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
gotData, gotParam, err := paramRoundTrip(c, test.val)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(gotData, test.wantData) {
|
|
||||||
t.Errorf("%#v:\ngot %#v (%T)\nwant %#v (%T)",
|
|
||||||
test.val, gotData, gotData, test.wantData, test.wantData)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(gotParam, test.wantParam) {
|
|
||||||
t.Errorf("%#v:\ngot %#v (%T)\nwant %#v (%T)",
|
|
||||||
test.val, gotParam, gotParam, test.wantParam, test.wantParam)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// paramRoundTrip passes x as a query parameter to BigQuery. It returns
|
|
||||||
// the resulting data value from running the query and the parameter value from
|
|
||||||
// the returned job configuration.
|
|
||||||
func paramRoundTrip(c *Client, x interface{}) (data Value, param interface{}, err error) {
|
|
||||||
ctx := context.Background()
|
|
||||||
q := c.Query("select ?")
|
|
||||||
q.Parameters = []QueryParameter{{Value: x}}
|
|
||||||
job, err := q.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
it, err := job.Read(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
var val []Value
|
|
||||||
err = it.Next(&val)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if len(val) != 1 {
|
|
||||||
return nil, nil, errors.New("wrong number of values")
|
|
||||||
}
|
|
||||||
conf, err := job.Config()
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
return val[0], conf.(*QueryConfig).Parameters[0].Value, nil
|
|
||||||
}
|
|
||||||
307
vendor/cloud.google.com/go/bigquery/query.go
generated
vendored
307
vendor/cloud.google.com/go/bigquery/query.go
generated
vendored
@ -1,307 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// QueryConfig holds the configuration for a query job.
|
|
||||||
type QueryConfig struct {
|
|
||||||
// Dst is the table into which the results of the query will be written.
|
|
||||||
// If this field is nil, a temporary table will be created.
|
|
||||||
Dst *Table
|
|
||||||
|
|
||||||
// The query to execute. See https://cloud.google.com/bigquery/query-reference for details.
|
|
||||||
Q string
|
|
||||||
|
|
||||||
// DefaultProjectID and DefaultDatasetID specify the dataset to use for unqualified table names in the query.
|
|
||||||
// If DefaultProjectID is set, DefaultDatasetID must also be set.
|
|
||||||
DefaultProjectID string
|
|
||||||
DefaultDatasetID string
|
|
||||||
|
|
||||||
// TableDefinitions describes data sources outside of BigQuery.
|
|
||||||
// The map keys may be used as table names in the query string.
|
|
||||||
//
|
|
||||||
// When a QueryConfig is returned from Job.Config, the map values
|
|
||||||
// are always of type *ExternalDataConfig.
|
|
||||||
TableDefinitions map[string]ExternalData
|
|
||||||
|
|
||||||
// CreateDisposition specifies the circumstances under which the destination table will be created.
|
|
||||||
// The default is CreateIfNeeded.
|
|
||||||
CreateDisposition TableCreateDisposition
|
|
||||||
|
|
||||||
// WriteDisposition specifies how existing data in the destination table is treated.
|
|
||||||
// The default is WriteEmpty.
|
|
||||||
WriteDisposition TableWriteDisposition
|
|
||||||
|
|
||||||
// DisableQueryCache prevents results being fetched from the query cache.
|
|
||||||
// If this field is false, results are fetched from the cache if they are available.
|
|
||||||
// The query cache is a best-effort cache that is flushed whenever tables in the query are modified.
|
|
||||||
// Cached results are only available when TableID is unspecified in the query's destination Table.
|
|
||||||
// For more information, see https://cloud.google.com/bigquery/querying-data#querycaching
|
|
||||||
DisableQueryCache bool
|
|
||||||
|
|
||||||
// DisableFlattenedResults prevents results being flattened.
|
|
||||||
// If this field is false, results from nested and repeated fields are flattened.
|
|
||||||
// DisableFlattenedResults implies AllowLargeResults
|
|
||||||
// For more information, see https://cloud.google.com/bigquery/docs/data#nested
|
|
||||||
DisableFlattenedResults bool
|
|
||||||
|
|
||||||
// AllowLargeResults allows the query to produce arbitrarily large result tables.
|
|
||||||
// The destination must be a table.
|
|
||||||
// When using this option, queries will take longer to execute, even if the result set is small.
|
|
||||||
// For additional limitations, see https://cloud.google.com/bigquery/querying-data#largequeryresults
|
|
||||||
AllowLargeResults bool
|
|
||||||
|
|
||||||
// Priority specifies the priority with which to schedule the query.
|
|
||||||
// The default priority is InteractivePriority.
|
|
||||||
// For more information, see https://cloud.google.com/bigquery/querying-data#batchqueries
|
|
||||||
Priority QueryPriority
|
|
||||||
|
|
||||||
// MaxBillingTier sets the maximum billing tier for a Query.
|
|
||||||
// Queries that have resource usage beyond this tier will fail (without
|
|
||||||
// incurring a charge). If this field is zero, the project default will be used.
|
|
||||||
MaxBillingTier int
|
|
||||||
|
|
||||||
// MaxBytesBilled limits the number of bytes billed for
|
|
||||||
// this job. Queries that would exceed this limit will fail (without incurring
|
|
||||||
// a charge).
|
|
||||||
// If this field is less than 1, the project default will be
|
|
||||||
// used.
|
|
||||||
MaxBytesBilled int64
|
|
||||||
|
|
||||||
// UseStandardSQL causes the query to use standard SQL. The default.
|
|
||||||
// Deprecated: use UseLegacySQL.
|
|
||||||
UseStandardSQL bool
|
|
||||||
|
|
||||||
// UseLegacySQL causes the query to use legacy SQL.
|
|
||||||
UseLegacySQL bool
|
|
||||||
|
|
||||||
// Parameters is a list of query parameters. The presence of parameters
|
|
||||||
// implies the use of standard SQL.
|
|
||||||
// If the query uses positional syntax ("?"), then no parameter may have a name.
|
|
||||||
// If the query uses named syntax ("@p"), then all parameters must have names.
|
|
||||||
// It is illegal to mix positional and named syntax.
|
|
||||||
Parameters []QueryParameter
|
|
||||||
|
|
||||||
// TimePartitioning specifies time-based partitioning
|
|
||||||
// for the destination table.
|
|
||||||
TimePartitioning *TimePartitioning
|
|
||||||
|
|
||||||
// The labels associated with this job.
|
|
||||||
Labels map[string]string
|
|
||||||
|
|
||||||
// If true, don't actually run this job. A valid query will return a mostly
|
|
||||||
// empty response with some processing statistics, while an invalid query will
|
|
||||||
// return the same error it would if it wasn't a dry run.
|
|
||||||
//
|
|
||||||
// Query.Read will fail with dry-run queries. Call Query.Run instead, and then
|
|
||||||
// call LastStatus on the returned job to get statistics. Calling Status on a
|
|
||||||
// dry-run job will fail.
|
|
||||||
DryRun bool
|
|
||||||
|
|
||||||
// Custom encryption configuration (e.g., Cloud KMS keys).
|
|
||||||
DestinationEncryptionConfig *EncryptionConfig
|
|
||||||
|
|
||||||
// Allows the schema of the destination table to be updated as a side effect of
|
|
||||||
// the query job.
|
|
||||||
SchemaUpdateOptions []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (qc *QueryConfig) toBQ() (*bq.JobConfiguration, error) {
|
|
||||||
qconf := &bq.JobConfigurationQuery{
|
|
||||||
Query: qc.Q,
|
|
||||||
CreateDisposition: string(qc.CreateDisposition),
|
|
||||||
WriteDisposition: string(qc.WriteDisposition),
|
|
||||||
AllowLargeResults: qc.AllowLargeResults,
|
|
||||||
Priority: string(qc.Priority),
|
|
||||||
MaximumBytesBilled: qc.MaxBytesBilled,
|
|
||||||
TimePartitioning: qc.TimePartitioning.toBQ(),
|
|
||||||
DestinationEncryptionConfiguration: qc.DestinationEncryptionConfig.toBQ(),
|
|
||||||
SchemaUpdateOptions: qc.SchemaUpdateOptions,
|
|
||||||
}
|
|
||||||
if len(qc.TableDefinitions) > 0 {
|
|
||||||
qconf.TableDefinitions = make(map[string]bq.ExternalDataConfiguration)
|
|
||||||
}
|
|
||||||
for name, data := range qc.TableDefinitions {
|
|
||||||
qconf.TableDefinitions[name] = data.toBQ()
|
|
||||||
}
|
|
||||||
if qc.DefaultProjectID != "" || qc.DefaultDatasetID != "" {
|
|
||||||
qconf.DefaultDataset = &bq.DatasetReference{
|
|
||||||
DatasetId: qc.DefaultDatasetID,
|
|
||||||
ProjectId: qc.DefaultProjectID,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if tier := int64(qc.MaxBillingTier); tier > 0 {
|
|
||||||
qconf.MaximumBillingTier = &tier
|
|
||||||
}
|
|
||||||
f := false
|
|
||||||
if qc.DisableQueryCache {
|
|
||||||
qconf.UseQueryCache = &f
|
|
||||||
}
|
|
||||||
if qc.DisableFlattenedResults {
|
|
||||||
qconf.FlattenResults = &f
|
|
||||||
// DisableFlattenResults implies AllowLargeResults.
|
|
||||||
qconf.AllowLargeResults = true
|
|
||||||
}
|
|
||||||
if qc.UseStandardSQL && qc.UseLegacySQL {
|
|
||||||
return nil, errors.New("bigquery: cannot provide both UseStandardSQL and UseLegacySQL")
|
|
||||||
}
|
|
||||||
if len(qc.Parameters) > 0 && qc.UseLegacySQL {
|
|
||||||
return nil, errors.New("bigquery: cannot provide both Parameters (implying standard SQL) and UseLegacySQL")
|
|
||||||
}
|
|
||||||
ptrue := true
|
|
||||||
pfalse := false
|
|
||||||
if qc.UseLegacySQL {
|
|
||||||
qconf.UseLegacySql = &ptrue
|
|
||||||
} else {
|
|
||||||
qconf.UseLegacySql = &pfalse
|
|
||||||
}
|
|
||||||
if qc.Dst != nil && !qc.Dst.implicitTable() {
|
|
||||||
qconf.DestinationTable = qc.Dst.toBQ()
|
|
||||||
}
|
|
||||||
for _, p := range qc.Parameters {
|
|
||||||
qp, err := p.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
qconf.QueryParameters = append(qconf.QueryParameters, qp)
|
|
||||||
}
|
|
||||||
return &bq.JobConfiguration{
|
|
||||||
Labels: qc.Labels,
|
|
||||||
DryRun: qc.DryRun,
|
|
||||||
Query: qconf,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToQueryConfig(q *bq.JobConfiguration, c *Client) (*QueryConfig, error) {
|
|
||||||
qq := q.Query
|
|
||||||
qc := &QueryConfig{
|
|
||||||
Labels: q.Labels,
|
|
||||||
DryRun: q.DryRun,
|
|
||||||
Q: qq.Query,
|
|
||||||
CreateDisposition: TableCreateDisposition(qq.CreateDisposition),
|
|
||||||
WriteDisposition: TableWriteDisposition(qq.WriteDisposition),
|
|
||||||
AllowLargeResults: qq.AllowLargeResults,
|
|
||||||
Priority: QueryPriority(qq.Priority),
|
|
||||||
MaxBytesBilled: qq.MaximumBytesBilled,
|
|
||||||
UseLegacySQL: qq.UseLegacySql == nil || *qq.UseLegacySql,
|
|
||||||
TimePartitioning: bqToTimePartitioning(qq.TimePartitioning),
|
|
||||||
DestinationEncryptionConfig: bqToEncryptionConfig(qq.DestinationEncryptionConfiguration),
|
|
||||||
SchemaUpdateOptions: qq.SchemaUpdateOptions,
|
|
||||||
}
|
|
||||||
qc.UseStandardSQL = !qc.UseLegacySQL
|
|
||||||
|
|
||||||
if len(qq.TableDefinitions) > 0 {
|
|
||||||
qc.TableDefinitions = make(map[string]ExternalData)
|
|
||||||
}
|
|
||||||
for name, qedc := range qq.TableDefinitions {
|
|
||||||
edc, err := bqToExternalDataConfig(&qedc)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
qc.TableDefinitions[name] = edc
|
|
||||||
}
|
|
||||||
if qq.DefaultDataset != nil {
|
|
||||||
qc.DefaultProjectID = qq.DefaultDataset.ProjectId
|
|
||||||
qc.DefaultDatasetID = qq.DefaultDataset.DatasetId
|
|
||||||
}
|
|
||||||
if qq.MaximumBillingTier != nil {
|
|
||||||
qc.MaxBillingTier = int(*qq.MaximumBillingTier)
|
|
||||||
}
|
|
||||||
if qq.UseQueryCache != nil && !*qq.UseQueryCache {
|
|
||||||
qc.DisableQueryCache = true
|
|
||||||
}
|
|
||||||
if qq.FlattenResults != nil && !*qq.FlattenResults {
|
|
||||||
qc.DisableFlattenedResults = true
|
|
||||||
}
|
|
||||||
if qq.DestinationTable != nil {
|
|
||||||
qc.Dst = bqToTable(qq.DestinationTable, c)
|
|
||||||
}
|
|
||||||
for _, qp := range qq.QueryParameters {
|
|
||||||
p, err := bqToQueryParameter(qp)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
qc.Parameters = append(qc.Parameters, p)
|
|
||||||
}
|
|
||||||
return qc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// QueryPriority specifies a priority with which a query is to be executed.
|
|
||||||
type QueryPriority string
|
|
||||||
|
|
||||||
const (
|
|
||||||
BatchPriority QueryPriority = "BATCH"
|
|
||||||
InteractivePriority QueryPriority = "INTERACTIVE"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A Query queries data from a BigQuery table. Use Client.Query to create a Query.
|
|
||||||
type Query struct {
|
|
||||||
JobIDConfig
|
|
||||||
QueryConfig
|
|
||||||
client *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// Query creates a query with string q.
|
|
||||||
// The returned Query may optionally be further configured before its Run method is called.
|
|
||||||
func (c *Client) Query(q string) *Query {
|
|
||||||
return &Query{
|
|
||||||
client: c,
|
|
||||||
QueryConfig: QueryConfig{Q: q},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run initiates a query job.
|
|
||||||
func (q *Query) Run(ctx context.Context) (j *Job, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Query.Run")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
job, err := q.newJob()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
j, err = q.client.insertJob(ctx, job, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return j, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *Query) newJob() (*bq.Job, error) {
|
|
||||||
config, err := q.QueryConfig.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: q.JobIDConfig.createJobRef(q.client),
|
|
||||||
Configuration: config,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read submits a query for execution and returns the results via a RowIterator.
|
|
||||||
// It is a shorthand for Query.Run followed by Job.Read.
|
|
||||||
func (q *Query) Read(ctx context.Context) (*RowIterator, error) {
|
|
||||||
job, err := q.Run(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return job.Read(ctx)
|
|
||||||
}
|
|
||||||
406
vendor/cloud.google.com/go/bigquery/query_test.go
generated
vendored
406
vendor/cloud.google.com/go/bigquery/query_test.go
generated
vendored
@ -1,406 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func defaultQueryJob() *bq.Job {
|
|
||||||
pfalse := false
|
|
||||||
return &bq.Job{
|
|
||||||
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
|
|
||||||
Configuration: &bq.JobConfiguration{
|
|
||||||
Query: &bq.JobConfigurationQuery{
|
|
||||||
DestinationTable: &bq.TableReference{
|
|
||||||
ProjectId: "client-project-id",
|
|
||||||
DatasetId: "dataset-id",
|
|
||||||
TableId: "table-id",
|
|
||||||
},
|
|
||||||
Query: "query string",
|
|
||||||
DefaultDataset: &bq.DatasetReference{
|
|
||||||
ProjectId: "def-project-id",
|
|
||||||
DatasetId: "def-dataset-id",
|
|
||||||
},
|
|
||||||
UseLegacySql: &pfalse,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var defaultQuery = &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuery(t *testing.T) {
|
|
||||||
defer fixRandomID("RANDOM")()
|
|
||||||
c := &Client{
|
|
||||||
projectID: "client-project-id",
|
|
||||||
}
|
|
||||||
testCases := []struct {
|
|
||||||
dst *Table
|
|
||||||
src *QueryConfig
|
|
||||||
jobIDConfig JobIDConfig
|
|
||||||
want *bq.Job
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: defaultQuery,
|
|
||||||
want: defaultQueryJob(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
DryRun: true,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Labels = map[string]string{"a": "b"}
|
|
||||||
j.Configuration.DryRun = true
|
|
||||||
j.Configuration.Query.DefaultDataset = nil
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
jobIDConfig: JobIDConfig{JobID: "jobID", AddJobIDSuffix: true},
|
|
||||||
src: &QueryConfig{Q: "query string"},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.DefaultDataset = nil
|
|
||||||
j.JobReference.JobId = "jobID-RANDOM"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: &Table{},
|
|
||||||
src: defaultQuery,
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.DestinationTable = nil
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
TableDefinitions: map[string]ExternalData{
|
|
||||||
"atable": func() *GCSReference {
|
|
||||||
g := NewGCSReference("uri")
|
|
||||||
g.AllowJaggedRows = true
|
|
||||||
g.AllowQuotedNewlines = true
|
|
||||||
g.Compression = Gzip
|
|
||||||
g.Encoding = UTF_8
|
|
||||||
g.FieldDelimiter = ";"
|
|
||||||
g.IgnoreUnknownValues = true
|
|
||||||
g.MaxBadRecords = 1
|
|
||||||
g.Quote = "'"
|
|
||||||
g.SkipLeadingRows = 2
|
|
||||||
g.Schema = Schema{{Name: "name", Type: StringFieldType}}
|
|
||||||
return g
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.DefaultDataset = nil
|
|
||||||
td := make(map[string]bq.ExternalDataConfiguration)
|
|
||||||
quote := "'"
|
|
||||||
td["atable"] = bq.ExternalDataConfiguration{
|
|
||||||
Compression: "GZIP",
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
MaxBadRecords: 1,
|
|
||||||
SourceFormat: "CSV", // must be explicitly set.
|
|
||||||
SourceUris: []string{"uri"},
|
|
||||||
CsvOptions: &bq.CsvOptions{
|
|
||||||
AllowJaggedRows: true,
|
|
||||||
AllowQuotedNewlines: true,
|
|
||||||
Encoding: "UTF-8",
|
|
||||||
FieldDelimiter: ";",
|
|
||||||
SkipLeadingRows: 2,
|
|
||||||
Quote: "e,
|
|
||||||
},
|
|
||||||
Schema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
{Name: "name", Type: "STRING"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
j.Configuration.Query.TableDefinitions = td
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: &Table{
|
|
||||||
ProjectID: "project-id",
|
|
||||||
DatasetID: "dataset-id",
|
|
||||||
TableID: "table-id",
|
|
||||||
},
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
CreateDisposition: CreateNever,
|
|
||||||
WriteDisposition: WriteTruncate,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.DestinationTable.ProjectId = "project-id"
|
|
||||||
j.Configuration.Query.WriteDisposition = "WRITE_TRUNCATE"
|
|
||||||
j.Configuration.Query.CreateDisposition = "CREATE_NEVER"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
DisableQueryCache: true,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
f := false
|
|
||||||
j.Configuration.Query.UseQueryCache = &f
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
AllowLargeResults: true,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.AllowLargeResults = true
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
DisableFlattenedResults: true,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
f := false
|
|
||||||
j.Configuration.Query.FlattenResults = &f
|
|
||||||
j.Configuration.Query.AllowLargeResults = true
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
Priority: QueryPriority("low"),
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
j.Configuration.Query.Priority = "low"
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
MaxBillingTier: 3,
|
|
||||||
MaxBytesBilled: 5,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
tier := int64(3)
|
|
||||||
j.Configuration.Query.MaximumBillingTier = &tier
|
|
||||||
j.Configuration.Query.MaximumBytesBilled = 5
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
UseStandardSQL: true,
|
|
||||||
},
|
|
||||||
want: defaultQueryJob(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
dst: c.Dataset("dataset-id").Table("table-id"),
|
|
||||||
src: &QueryConfig{
|
|
||||||
Q: "query string",
|
|
||||||
DefaultProjectID: "def-project-id",
|
|
||||||
DefaultDatasetID: "def-dataset-id",
|
|
||||||
UseLegacySQL: true,
|
|
||||||
},
|
|
||||||
want: func() *bq.Job {
|
|
||||||
j := defaultQueryJob()
|
|
||||||
ptrue := true
|
|
||||||
j.Configuration.Query.UseLegacySql = &ptrue
|
|
||||||
j.Configuration.Query.ForceSendFields = nil
|
|
||||||
return j
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for i, tc := range testCases {
|
|
||||||
query := c.Query("")
|
|
||||||
query.JobIDConfig = tc.jobIDConfig
|
|
||||||
query.QueryConfig = *tc.src
|
|
||||||
query.Dst = tc.dst
|
|
||||||
got, err := query.newJob()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("#%d: err calling query: %v", i, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
checkJob(t, i, got, tc.want)
|
|
||||||
|
|
||||||
// Round-trip.
|
|
||||||
jc, err := bqToJobConfig(got.Configuration, c)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("#%d: %v", i, err)
|
|
||||||
}
|
|
||||||
wantConfig := query.QueryConfig
|
|
||||||
// We set AllowLargeResults to true when DisableFlattenedResults is true.
|
|
||||||
if wantConfig.DisableFlattenedResults {
|
|
||||||
wantConfig.AllowLargeResults = true
|
|
||||||
}
|
|
||||||
// A QueryConfig with neither UseXXXSQL field set is equivalent
|
|
||||||
// to one where UseStandardSQL = true.
|
|
||||||
if !wantConfig.UseLegacySQL && !wantConfig.UseStandardSQL {
|
|
||||||
wantConfig.UseStandardSQL = true
|
|
||||||
}
|
|
||||||
// Treat nil and empty tables the same, and ignore the client.
|
|
||||||
tableEqual := func(t1, t2 *Table) bool {
|
|
||||||
if t1 == nil {
|
|
||||||
t1 = &Table{}
|
|
||||||
}
|
|
||||||
if t2 == nil {
|
|
||||||
t2 = &Table{}
|
|
||||||
}
|
|
||||||
return t1.ProjectID == t2.ProjectID && t1.DatasetID == t2.DatasetID && t1.TableID == t2.TableID
|
|
||||||
}
|
|
||||||
// A table definition that is a GCSReference round-trips as an ExternalDataConfig.
|
|
||||||
// TODO(jba): see if there is a way to express this with a transformer.
|
|
||||||
gcsRefToEDC := func(g *GCSReference) *ExternalDataConfig {
|
|
||||||
q := g.toBQ()
|
|
||||||
e, _ := bqToExternalDataConfig(&q)
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
externalDataEqual := func(e1, e2 ExternalData) bool {
|
|
||||||
if r, ok := e1.(*GCSReference); ok {
|
|
||||||
e1 = gcsRefToEDC(r)
|
|
||||||
}
|
|
||||||
if r, ok := e2.(*GCSReference); ok {
|
|
||||||
e2 = gcsRefToEDC(r)
|
|
||||||
}
|
|
||||||
return cmp.Equal(e1, e2)
|
|
||||||
}
|
|
||||||
diff := testutil.Diff(jc.(*QueryConfig), &wantConfig,
|
|
||||||
cmp.Comparer(tableEqual),
|
|
||||||
cmp.Comparer(externalDataEqual),
|
|
||||||
)
|
|
||||||
if diff != "" {
|
|
||||||
t.Errorf("#%d: (got=-, want=+:\n%s", i, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguringQuery(t *testing.T) {
|
|
||||||
c := &Client{
|
|
||||||
projectID: "project-id",
|
|
||||||
}
|
|
||||||
|
|
||||||
query := c.Query("q")
|
|
||||||
query.JobID = "ajob"
|
|
||||||
query.DefaultProjectID = "def-project-id"
|
|
||||||
query.DefaultDatasetID = "def-dataset-id"
|
|
||||||
query.TimePartitioning = &TimePartitioning{Expiration: 1234 * time.Second, Field: "f"}
|
|
||||||
query.DestinationEncryptionConfig = &EncryptionConfig{KMSKeyName: "keyName"}
|
|
||||||
query.SchemaUpdateOptions = []string{"ALLOW_FIELD_ADDITION"}
|
|
||||||
|
|
||||||
// Note: Other configuration fields are tested in other tests above.
|
|
||||||
// A lot of that can be consolidated once Client.Copy is gone.
|
|
||||||
|
|
||||||
pfalse := false
|
|
||||||
want := &bq.Job{
|
|
||||||
Configuration: &bq.JobConfiguration{
|
|
||||||
Query: &bq.JobConfigurationQuery{
|
|
||||||
Query: "q",
|
|
||||||
DefaultDataset: &bq.DatasetReference{
|
|
||||||
ProjectId: "def-project-id",
|
|
||||||
DatasetId: "def-dataset-id",
|
|
||||||
},
|
|
||||||
UseLegacySql: &pfalse,
|
|
||||||
TimePartitioning: &bq.TimePartitioning{ExpirationMs: 1234000, Field: "f", Type: "DAY"},
|
|
||||||
DestinationEncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
|
|
||||||
SchemaUpdateOptions: []string{"ALLOW_FIELD_ADDITION"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
JobReference: &bq.JobReference{
|
|
||||||
JobId: "ajob",
|
|
||||||
ProjectId: "project-id",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := query.newJob()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err calling Query.newJob: %v", err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Errorf("querying: -got +want:\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQueryLegacySQL(t *testing.T) {
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
q := c.Query("q")
|
|
||||||
q.UseStandardSQL = true
|
|
||||||
q.UseLegacySQL = true
|
|
||||||
_, err := q.newJob()
|
|
||||||
if err == nil {
|
|
||||||
t.Error("UseStandardSQL and UseLegacySQL: got nil, want error")
|
|
||||||
}
|
|
||||||
q = c.Query("q")
|
|
||||||
q.Parameters = []QueryParameter{{Name: "p", Value: 3}}
|
|
||||||
q.UseLegacySQL = true
|
|
||||||
_, err = q.newJob()
|
|
||||||
if err == nil {
|
|
||||||
t.Error("Parameters and UseLegacySQL: got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
235
vendor/cloud.google.com/go/bigquery/read_test.go
generated
vendored
235
vendor/cloud.google.com/go/bigquery/read_test.go
generated
vendored
@ -1,235 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
)
|
|
||||||
|
|
||||||
type pageFetcherArgs struct {
|
|
||||||
table *Table
|
|
||||||
schema Schema
|
|
||||||
startIndex uint64
|
|
||||||
pageSize int64
|
|
||||||
pageToken string
|
|
||||||
}
|
|
||||||
|
|
||||||
// pageFetcherReadStub services read requests by returning data from an in-memory list of values.
|
|
||||||
type pageFetcherReadStub struct {
|
|
||||||
// values and pageTokens are used as sources of data to return in response to calls to readTabledata or readQuery.
|
|
||||||
values [][][]Value // contains pages / rows / columns.
|
|
||||||
pageTokens map[string]string // maps incoming page token to returned page token.
|
|
||||||
|
|
||||||
// arguments are recorded for later inspection.
|
|
||||||
calls []pageFetcherArgs
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *pageFetcherReadStub) fetchPage(ctx context.Context, t *Table, schema Schema, startIndex uint64, pageSize int64, pageToken string) (*fetchPageResult, error) {
|
|
||||||
s.calls = append(s.calls,
|
|
||||||
pageFetcherArgs{t, schema, startIndex, pageSize, pageToken})
|
|
||||||
result := &fetchPageResult{
|
|
||||||
pageToken: s.pageTokens[pageToken],
|
|
||||||
rows: s.values[0],
|
|
||||||
}
|
|
||||||
s.values = s.values[1:]
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForQueryStub(context.Context, string) (Schema, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRead(t *testing.T) {
|
|
||||||
// The data for the service stub to return is populated for each test case in the testCases for loop.
|
|
||||||
ctx := context.Background()
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
pf := &pageFetcherReadStub{}
|
|
||||||
queryJob := &Job{
|
|
||||||
projectID: "project-id",
|
|
||||||
jobID: "job-id",
|
|
||||||
c: c,
|
|
||||||
config: &bq.JobConfiguration{
|
|
||||||
Query: &bq.JobConfigurationQuery{
|
|
||||||
DestinationTable: &bq.TableReference{
|
|
||||||
ProjectId: "project-id",
|
|
||||||
DatasetId: "dataset-id",
|
|
||||||
TableId: "table-id",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, readFunc := range []func() *RowIterator{
|
|
||||||
func() *RowIterator {
|
|
||||||
return c.Dataset("dataset-id").Table("table-id").read(ctx, pf.fetchPage)
|
|
||||||
},
|
|
||||||
func() *RowIterator {
|
|
||||||
it, err := queryJob.read(ctx, waitForQueryStub, pf.fetchPage)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return it
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
testCases := []struct {
|
|
||||||
data [][][]Value
|
|
||||||
pageTokens map[string]string
|
|
||||||
want [][]Value
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}},
|
|
||||||
pageTokens: map[string]string{"": "a", "a": ""},
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}, {30, 40}, {31, 41}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}},
|
|
||||||
pageTokens: map[string]string{"": ""}, // no more pages after first one.
|
|
||||||
want: [][]Value{{1, 2}, {11, 12}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range testCases {
|
|
||||||
pf.values = tc.data
|
|
||||||
pf.pageTokens = tc.pageTokens
|
|
||||||
if got, ok := collectValues(t, readFunc()); ok {
|
|
||||||
if !testutil.Equal(got, tc.want) {
|
|
||||||
t.Errorf("reading: got:\n%v\nwant:\n%v", got, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectValues(t *testing.T, it *RowIterator) ([][]Value, bool) {
|
|
||||||
var got [][]Value
|
|
||||||
for {
|
|
||||||
var vals []Value
|
|
||||||
err := it.Next(&vals)
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("err calling Next: %v", err)
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
got = append(got, vals)
|
|
||||||
}
|
|
||||||
return got, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoMoreValues(t *testing.T) {
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
pf := &pageFetcherReadStub{
|
|
||||||
values: [][][]Value{{{1, 2}, {11, 12}}},
|
|
||||||
}
|
|
||||||
it := c.Dataset("dataset-id").Table("table-id").read(context.Background(), pf.fetchPage)
|
|
||||||
var vals []Value
|
|
||||||
// We expect to retrieve two values and then fail on the next attempt.
|
|
||||||
if err := it.Next(&vals); err != nil {
|
|
||||||
t.Fatalf("Next: got: %v: want: nil", err)
|
|
||||||
}
|
|
||||||
if err := it.Next(&vals); err != nil {
|
|
||||||
t.Fatalf("Next: got: %v: want: nil", err)
|
|
||||||
}
|
|
||||||
if err := it.Next(&vals); err != iterator.Done {
|
|
||||||
t.Fatalf("Next: got: %v: want: iterator.Done", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var errBang = errors.New("bang!")
|
|
||||||
|
|
||||||
func errorFetchPage(context.Context, *Table, Schema, uint64, int64, string) (*fetchPageResult, error) {
|
|
||||||
return nil, errBang
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadError(t *testing.T) {
|
|
||||||
// test that service read errors are propagated back to the caller.
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
it := c.Dataset("dataset-id").Table("table-id").read(context.Background(), errorFetchPage)
|
|
||||||
var vals []Value
|
|
||||||
if err := it.Next(&vals); err != errBang {
|
|
||||||
t.Fatalf("Get: got: %v: want: %v", err, errBang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadTabledataOptions(t *testing.T) {
|
|
||||||
// test that read options are propagated.
|
|
||||||
s := &pageFetcherReadStub{
|
|
||||||
values: [][][]Value{{{1, 2}}},
|
|
||||||
}
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
tr := c.Dataset("dataset-id").Table("table-id")
|
|
||||||
it := tr.read(context.Background(), s.fetchPage)
|
|
||||||
it.PageInfo().MaxSize = 5
|
|
||||||
var vals []Value
|
|
||||||
if err := it.Next(&vals); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := []pageFetcherArgs{{
|
|
||||||
table: tr,
|
|
||||||
pageSize: 5,
|
|
||||||
pageToken: "",
|
|
||||||
}}
|
|
||||||
if diff := testutil.Diff(s.calls, want, cmp.AllowUnexported(pageFetcherArgs{}, pageFetcherReadStub{}, Table{}, Client{})); diff != "" {
|
|
||||||
t.Errorf("reading (got=-, want=+):\n%s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadQueryOptions(t *testing.T) {
|
|
||||||
// test that read options are propagated.
|
|
||||||
c := &Client{projectID: "project-id"}
|
|
||||||
pf := &pageFetcherReadStub{
|
|
||||||
values: [][][]Value{{{1, 2}}},
|
|
||||||
}
|
|
||||||
tr := &bq.TableReference{
|
|
||||||
ProjectId: "project-id",
|
|
||||||
DatasetId: "dataset-id",
|
|
||||||
TableId: "table-id",
|
|
||||||
}
|
|
||||||
queryJob := &Job{
|
|
||||||
projectID: "project-id",
|
|
||||||
jobID: "job-id",
|
|
||||||
c: c,
|
|
||||||
config: &bq.JobConfiguration{
|
|
||||||
Query: &bq.JobConfigurationQuery{DestinationTable: tr},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
it, err := queryJob.read(context.Background(), waitForQueryStub, pf.fetchPage)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err calling Read: %v", err)
|
|
||||||
}
|
|
||||||
it.PageInfo().MaxSize = 5
|
|
||||||
var vals []Value
|
|
||||||
if err := it.Next(&vals); err != nil {
|
|
||||||
t.Fatalf("Next: got: %v: want: nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []pageFetcherArgs{{
|
|
||||||
table: bqToTable(tr, c),
|
|
||||||
pageSize: 5,
|
|
||||||
pageToken: "",
|
|
||||||
}}
|
|
||||||
if !testutil.Equal(pf.calls, want, cmp.AllowUnexported(pageFetcherArgs{}, Table{}, Client{})) {
|
|
||||||
t.Errorf("reading: got:\n%v\nwant:\n%v", pf.calls, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
397
vendor/cloud.google.com/go/bigquery/schema.go
generated
vendored
397
vendor/cloud.google.com/go/bigquery/schema.go
generated
vendored
@ -1,397 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/atomiccache"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Schema describes the fields in a table or query result.
|
|
||||||
type Schema []*FieldSchema
|
|
||||||
|
|
||||||
type FieldSchema struct {
|
|
||||||
// The field name.
|
|
||||||
// Must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
|
|
||||||
// and must start with a letter or underscore.
|
|
||||||
// The maximum length is 128 characters.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// A description of the field. The maximum length is 16,384 characters.
|
|
||||||
Description string
|
|
||||||
|
|
||||||
// Whether the field may contain multiple values.
|
|
||||||
Repeated bool
|
|
||||||
// Whether the field is required. Ignored if Repeated is true.
|
|
||||||
Required bool
|
|
||||||
|
|
||||||
// The field data type. If Type is Record, then this field contains a nested schema,
|
|
||||||
// which is described by Schema.
|
|
||||||
Type FieldType
|
|
||||||
// Describes the nested schema if Type is set to Record.
|
|
||||||
Schema Schema
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fs *FieldSchema) toBQ() *bq.TableFieldSchema {
|
|
||||||
tfs := &bq.TableFieldSchema{
|
|
||||||
Description: fs.Description,
|
|
||||||
Name: fs.Name,
|
|
||||||
Type: string(fs.Type),
|
|
||||||
}
|
|
||||||
|
|
||||||
if fs.Repeated {
|
|
||||||
tfs.Mode = "REPEATED"
|
|
||||||
} else if fs.Required {
|
|
||||||
tfs.Mode = "REQUIRED"
|
|
||||||
} // else leave as default, which is interpreted as NULLABLE.
|
|
||||||
|
|
||||||
for _, f := range fs.Schema {
|
|
||||||
tfs.Fields = append(tfs.Fields, f.toBQ())
|
|
||||||
}
|
|
||||||
|
|
||||||
return tfs
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s Schema) toBQ() *bq.TableSchema {
|
|
||||||
var fields []*bq.TableFieldSchema
|
|
||||||
for _, f := range s {
|
|
||||||
fields = append(fields, f.toBQ())
|
|
||||||
}
|
|
||||||
return &bq.TableSchema{Fields: fields}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
|
|
||||||
fs := &FieldSchema{
|
|
||||||
Description: tfs.Description,
|
|
||||||
Name: tfs.Name,
|
|
||||||
Repeated: tfs.Mode == "REPEATED",
|
|
||||||
Required: tfs.Mode == "REQUIRED",
|
|
||||||
Type: FieldType(tfs.Type),
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range tfs.Fields {
|
|
||||||
fs.Schema = append(fs.Schema, bqToFieldSchema(f))
|
|
||||||
}
|
|
||||||
return fs
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToSchema(ts *bq.TableSchema) Schema {
|
|
||||||
if ts == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var s Schema
|
|
||||||
for _, f := range ts.Fields {
|
|
||||||
s = append(s, bqToFieldSchema(f))
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
type FieldType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
StringFieldType FieldType = "STRING"
|
|
||||||
BytesFieldType FieldType = "BYTES"
|
|
||||||
IntegerFieldType FieldType = "INTEGER"
|
|
||||||
FloatFieldType FieldType = "FLOAT"
|
|
||||||
BooleanFieldType FieldType = "BOOLEAN"
|
|
||||||
TimestampFieldType FieldType = "TIMESTAMP"
|
|
||||||
RecordFieldType FieldType = "RECORD"
|
|
||||||
DateFieldType FieldType = "DATE"
|
|
||||||
TimeFieldType FieldType = "TIME"
|
|
||||||
DateTimeFieldType FieldType = "DATETIME"
|
|
||||||
NumericFieldType FieldType = "NUMERIC"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errNoStruct = errors.New("bigquery: can only infer schema from struct or pointer to struct")
|
|
||||||
errUnsupportedFieldType = errors.New("bigquery: unsupported type of field in struct")
|
|
||||||
errInvalidFieldName = errors.New("bigquery: invalid name of field in struct")
|
|
||||||
errBadNullable = errors.New(`bigquery: use "nullable" only for []byte and struct pointers; for all other types, use a NullXXX type`)
|
|
||||||
)
|
|
||||||
|
|
||||||
var typeOfByteSlice = reflect.TypeOf([]byte{})
|
|
||||||
|
|
||||||
// InferSchema tries to derive a BigQuery schema from the supplied struct value.
|
|
||||||
// Each exported struct field is mapped to a field in the schema.
|
|
||||||
//
|
|
||||||
// The following BigQuery types are inferred from the corresponding Go types.
|
|
||||||
// (This is the same mapping as that used for RowIterator.Next.) Fields inferred
|
|
||||||
// from these types are marked required (non-nullable).
|
|
||||||
//
|
|
||||||
// STRING string
|
|
||||||
// BOOL bool
|
|
||||||
// INTEGER int, int8, int16, int32, int64, uint8, uint16, uint32
|
|
||||||
// FLOAT float32, float64
|
|
||||||
// BYTES []byte
|
|
||||||
// TIMESTAMP time.Time
|
|
||||||
// DATE civil.Date
|
|
||||||
// TIME civil.Time
|
|
||||||
// DATETIME civil.DateTime
|
|
||||||
// NUMERIC *big.Rat
|
|
||||||
//
|
|
||||||
// The big.Rat type supports numbers of arbitrary size and precision. Values
|
|
||||||
// will be rounded to 9 digits after the decimal point before being transmitted
|
|
||||||
// to BigQuery. See https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#numeric-type
|
|
||||||
// for more on NUMERIC.
|
|
||||||
//
|
|
||||||
// A Go slice or array type is inferred to be a BigQuery repeated field of the
|
|
||||||
// element type. The element type must be one of the above listed types.
|
|
||||||
//
|
|
||||||
// Nullable fields are inferred from the NullXXX types, declared in this package:
|
|
||||||
//
|
|
||||||
// STRING NullString
|
|
||||||
// BOOL NullBool
|
|
||||||
// INTEGER NullInt64
|
|
||||||
// FLOAT NullFloat64
|
|
||||||
// TIMESTAMP NullTimestamp
|
|
||||||
// DATE NullDate
|
|
||||||
// TIME NullTime
|
|
||||||
// DATETIME NullDateTime
|
|
||||||
//
|
|
||||||
// For a nullable BYTES field, use the type []byte and tag the field "nullable" (see below).
|
|
||||||
// For a nullable NUMERIC field, use the type *big.Rat and tag the field "nullable".
|
|
||||||
//
|
|
||||||
// A struct field that is of struct type is inferred to be a required field of type
|
|
||||||
// RECORD with a schema inferred recursively. For backwards compatibility, a field of
|
|
||||||
// type pointer to struct is also inferred to be required. To get a nullable RECORD
|
|
||||||
// field, use the "nullable" tag (see below).
|
|
||||||
//
|
|
||||||
// InferSchema returns an error if any of the examined fields is of type uint,
|
|
||||||
// uint64, uintptr, map, interface, complex64, complex128, func, or chan. Future
|
|
||||||
// versions may handle these cases without error.
|
|
||||||
//
|
|
||||||
// Recursively defined structs are also disallowed.
|
|
||||||
//
|
|
||||||
// Struct fields may be tagged in a way similar to the encoding/json package.
|
|
||||||
// A tag of the form
|
|
||||||
// bigquery:"name"
|
|
||||||
// uses "name" instead of the struct field name as the BigQuery field name.
|
|
||||||
// A tag of the form
|
|
||||||
// bigquery:"-"
|
|
||||||
// omits the field from the inferred schema.
|
|
||||||
// The "nullable" option marks the field as nullable (not required). It is only
|
|
||||||
// needed for []byte, *big.Rat and pointer-to-struct fields, and cannot appear on other
|
|
||||||
// fields. In this example, the Go name of the field is retained:
|
|
||||||
// bigquery:",nullable"
|
|
||||||
func InferSchema(st interface{}) (Schema, error) {
|
|
||||||
return inferSchemaReflectCached(reflect.TypeOf(st))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO(jba): replace with sync.Map for Go 1.9.
|
|
||||||
var schemaCache atomiccache.Cache
|
|
||||||
|
|
||||||
type cacheVal struct {
|
|
||||||
schema Schema
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func inferSchemaReflectCached(t reflect.Type) (Schema, error) {
|
|
||||||
cv := schemaCache.Get(t, func() interface{} {
|
|
||||||
s, err := inferSchemaReflect(t)
|
|
||||||
return cacheVal{s, err}
|
|
||||||
}).(cacheVal)
|
|
||||||
return cv.schema, cv.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func inferSchemaReflect(t reflect.Type) (Schema, error) {
|
|
||||||
rec, err := hasRecursiveType(t, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if rec {
|
|
||||||
return nil, fmt.Errorf("bigquery: schema inference for recursive type %s", t)
|
|
||||||
}
|
|
||||||
return inferStruct(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func inferStruct(t reflect.Type) (Schema, error) {
|
|
||||||
switch t.Kind() {
|
|
||||||
case reflect.Ptr:
|
|
||||||
if t.Elem().Kind() != reflect.Struct {
|
|
||||||
return nil, errNoStruct
|
|
||||||
}
|
|
||||||
t = t.Elem()
|
|
||||||
fallthrough
|
|
||||||
|
|
||||||
case reflect.Struct:
|
|
||||||
return inferFields(t)
|
|
||||||
default:
|
|
||||||
return nil, errNoStruct
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// inferFieldSchema infers the FieldSchema for a Go type
|
|
||||||
func inferFieldSchema(rt reflect.Type, nullable bool) (*FieldSchema, error) {
|
|
||||||
// Only []byte and struct pointers can be tagged nullable.
|
|
||||||
if nullable && !(rt == typeOfByteSlice || rt.Kind() == reflect.Ptr && rt.Elem().Kind() == reflect.Struct) {
|
|
||||||
return nil, errBadNullable
|
|
||||||
}
|
|
||||||
switch rt {
|
|
||||||
case typeOfByteSlice:
|
|
||||||
return &FieldSchema{Required: !nullable, Type: BytesFieldType}, nil
|
|
||||||
case typeOfGoTime:
|
|
||||||
return &FieldSchema{Required: true, Type: TimestampFieldType}, nil
|
|
||||||
case typeOfDate:
|
|
||||||
return &FieldSchema{Required: true, Type: DateFieldType}, nil
|
|
||||||
case typeOfTime:
|
|
||||||
return &FieldSchema{Required: true, Type: TimeFieldType}, nil
|
|
||||||
case typeOfDateTime:
|
|
||||||
return &FieldSchema{Required: true, Type: DateTimeFieldType}, nil
|
|
||||||
case typeOfRat:
|
|
||||||
return &FieldSchema{Required: !nullable, Type: NumericFieldType}, nil
|
|
||||||
}
|
|
||||||
if ft := nullableFieldType(rt); ft != "" {
|
|
||||||
return &FieldSchema{Required: false, Type: ft}, nil
|
|
||||||
}
|
|
||||||
if isSupportedIntType(rt) || isSupportedUintType(rt) {
|
|
||||||
return &FieldSchema{Required: true, Type: IntegerFieldType}, nil
|
|
||||||
}
|
|
||||||
switch rt.Kind() {
|
|
||||||
case reflect.Slice, reflect.Array:
|
|
||||||
et := rt.Elem()
|
|
||||||
if et != typeOfByteSlice && (et.Kind() == reflect.Slice || et.Kind() == reflect.Array) {
|
|
||||||
// Multi dimensional slices/arrays are not supported by BigQuery
|
|
||||||
return nil, errUnsupportedFieldType
|
|
||||||
}
|
|
||||||
if nullableFieldType(et) != "" {
|
|
||||||
// Repeated nullable types are not supported by BigQuery.
|
|
||||||
return nil, errUnsupportedFieldType
|
|
||||||
}
|
|
||||||
f, err := inferFieldSchema(et, false)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
f.Repeated = true
|
|
||||||
f.Required = false
|
|
||||||
return f, nil
|
|
||||||
case reflect.Ptr:
|
|
||||||
if rt.Elem().Kind() != reflect.Struct {
|
|
||||||
return nil, errUnsupportedFieldType
|
|
||||||
}
|
|
||||||
fallthrough
|
|
||||||
case reflect.Struct:
|
|
||||||
nested, err := inferStruct(rt)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &FieldSchema{Required: !nullable, Type: RecordFieldType, Schema: nested}, nil
|
|
||||||
case reflect.String:
|
|
||||||
return &FieldSchema{Required: !nullable, Type: StringFieldType}, nil
|
|
||||||
case reflect.Bool:
|
|
||||||
return &FieldSchema{Required: !nullable, Type: BooleanFieldType}, nil
|
|
||||||
case reflect.Float32, reflect.Float64:
|
|
||||||
return &FieldSchema{Required: !nullable, Type: FloatFieldType}, nil
|
|
||||||
default:
|
|
||||||
return nil, errUnsupportedFieldType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// inferFields extracts all exported field types from struct type.
|
|
||||||
func inferFields(rt reflect.Type) (Schema, error) {
|
|
||||||
var s Schema
|
|
||||||
fields, err := fieldCache.Fields(rt)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, field := range fields {
|
|
||||||
var nullable bool
|
|
||||||
for _, opt := range field.ParsedTag.([]string) {
|
|
||||||
if opt == nullableTagOption {
|
|
||||||
nullable = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f, err := inferFieldSchema(field.Type, nullable)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
f.Name = field.Name
|
|
||||||
s = append(s, f)
|
|
||||||
}
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// isSupportedIntType reports whether t is an int type that can be properly
|
|
||||||
// represented by the BigQuery INTEGER/INT64 type.
|
|
||||||
func isSupportedIntType(t reflect.Type) bool {
|
|
||||||
switch t.Kind() {
|
|
||||||
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// isSupportedIntType reports whether t is a uint type that can be properly
|
|
||||||
// represented by the BigQuery INTEGER/INT64 type.
|
|
||||||
func isSupportedUintType(t reflect.Type) bool {
|
|
||||||
switch t.Kind() {
|
|
||||||
case reflect.Uint8, reflect.Uint16, reflect.Uint32:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// typeList is a linked list of reflect.Types.
|
|
||||||
type typeList struct {
|
|
||||||
t reflect.Type
|
|
||||||
next *typeList
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *typeList) has(t reflect.Type) bool {
|
|
||||||
for l != nil {
|
|
||||||
if l.t == t {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
l = l.next
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// hasRecursiveType reports whether t or any type inside t refers to itself, directly or indirectly,
|
|
||||||
// via exported fields. (Schema inference ignores unexported fields.)
|
|
||||||
func hasRecursiveType(t reflect.Type, seen *typeList) (bool, error) {
|
|
||||||
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array {
|
|
||||||
t = t.Elem()
|
|
||||||
}
|
|
||||||
if t.Kind() != reflect.Struct {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
if seen.has(t) {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
fields, err := fieldCache.Fields(t)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
seen = &typeList{t, seen}
|
|
||||||
// Because seen is a linked list, additions to it from one field's
|
|
||||||
// recursive call will not affect the value for subsequent fields' calls.
|
|
||||||
for _, field := range fields {
|
|
||||||
ok, err := hasRecursiveType(field.Type, seen)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if ok {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
920
vendor/cloud.google.com/go/bigquery/schema_test.go
generated
vendored
920
vendor/cloud.google.com/go/bigquery/schema_test.go
generated
vendored
@ -1,920 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"math/big"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
"cloud.google.com/go/internal/pretty"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (fs *FieldSchema) GoString() string {
|
|
||||||
if fs == nil {
|
|
||||||
return "<nil>"
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}",
|
|
||||||
fs.Name,
|
|
||||||
fs.Description,
|
|
||||||
fs.Repeated,
|
|
||||||
fs.Required,
|
|
||||||
fs.Type,
|
|
||||||
fmt.Sprintf("%#v", fs.Schema),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema {
|
|
||||||
return &bq.TableFieldSchema{
|
|
||||||
Description: desc,
|
|
||||||
Name: name,
|
|
||||||
Mode: mode,
|
|
||||||
Type: typ,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema {
|
|
||||||
return &FieldSchema{
|
|
||||||
Description: desc,
|
|
||||||
Name: name,
|
|
||||||
Repeated: repeated,
|
|
||||||
Required: required,
|
|
||||||
Type: FieldType(typ),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSchemaConversion(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
schema Schema
|
|
||||||
bqSchema *bq.TableSchema
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
// required
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "STRING", false, true),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// repeated
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "STRING", "REPEATED"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "STRING", true, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// nullable, string
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "STRING", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "STRING", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// integer
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "INTEGER", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "INTEGER", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// float
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "FLOAT", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "FLOAT", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// boolean
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "BOOLEAN", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "BOOLEAN", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// timestamp
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "TIMESTAMP", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "name", "TIMESTAMP", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// civil times
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "f1", "TIME", ""),
|
|
||||||
bqTableFieldSchema("desc", "f2", "DATE", ""),
|
|
||||||
bqTableFieldSchema("desc", "f3", "DATETIME", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "f1", "TIME", false, false),
|
|
||||||
fieldSchema("desc", "f2", "DATE", false, false),
|
|
||||||
fieldSchema("desc", "f3", "DATETIME", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// numeric
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "n", "NUMERIC", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
fieldSchema("desc", "n", "NUMERIC", false, false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// nested
|
|
||||||
bqSchema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
{
|
|
||||||
Description: "An outer schema wrapping a nested schema",
|
|
||||||
Name: "outer",
|
|
||||||
Mode: "REQUIRED",
|
|
||||||
Type: "RECORD",
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("inner field", "inner", "STRING", ""),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
schema: Schema{
|
|
||||||
&FieldSchema{
|
|
||||||
Description: "An outer schema wrapping a nested schema",
|
|
||||||
Name: "outer",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{
|
|
||||||
{
|
|
||||||
Description: "inner field",
|
|
||||||
Name: "inner",
|
|
||||||
Type: "STRING",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range testCases {
|
|
||||||
bqSchema := tc.schema.toBQ()
|
|
||||||
if !testutil.Equal(bqSchema, tc.bqSchema) {
|
|
||||||
t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v",
|
|
||||||
pretty.Value(bqSchema), pretty.Value(tc.bqSchema))
|
|
||||||
}
|
|
||||||
schema := bqToSchema(tc.bqSchema)
|
|
||||||
if !testutil.Equal(schema, tc.schema) {
|
|
||||||
t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type allStrings struct {
|
|
||||||
String string
|
|
||||||
ByteSlice []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
type allSignedIntegers struct {
|
|
||||||
Int64 int64
|
|
||||||
Int32 int32
|
|
||||||
Int16 int16
|
|
||||||
Int8 int8
|
|
||||||
Int int
|
|
||||||
}
|
|
||||||
|
|
||||||
type allUnsignedIntegers struct {
|
|
||||||
Uint32 uint32
|
|
||||||
Uint16 uint16
|
|
||||||
Uint8 uint8
|
|
||||||
}
|
|
||||||
|
|
||||||
type allFloat struct {
|
|
||||||
Float64 float64
|
|
||||||
Float32 float32
|
|
||||||
// NOTE: Complex32 and Complex64 are unsupported by BigQuery
|
|
||||||
}
|
|
||||||
|
|
||||||
type allBoolean struct {
|
|
||||||
Bool bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type allTime struct {
|
|
||||||
Timestamp time.Time
|
|
||||||
Time civil.Time
|
|
||||||
Date civil.Date
|
|
||||||
DateTime civil.DateTime
|
|
||||||
}
|
|
||||||
|
|
||||||
type allNumeric struct {
|
|
||||||
Numeric *big.Rat
|
|
||||||
}
|
|
||||||
|
|
||||||
func reqField(name, typ string) *FieldSchema {
|
|
||||||
return &FieldSchema{
|
|
||||||
Name: name,
|
|
||||||
Type: FieldType(typ),
|
|
||||||
Required: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func optField(name, typ string) *FieldSchema {
|
|
||||||
return &FieldSchema{
|
|
||||||
Name: name,
|
|
||||||
Type: FieldType(typ),
|
|
||||||
Required: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSimpleInference(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
want Schema
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: allSignedIntegers{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Int64", "INTEGER"),
|
|
||||||
reqField("Int32", "INTEGER"),
|
|
||||||
reqField("Int16", "INTEGER"),
|
|
||||||
reqField("Int8", "INTEGER"),
|
|
||||||
reqField("Int", "INTEGER"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: allUnsignedIntegers{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Uint32", "INTEGER"),
|
|
||||||
reqField("Uint16", "INTEGER"),
|
|
||||||
reqField("Uint8", "INTEGER"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: allFloat{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Float64", "FLOAT"),
|
|
||||||
reqField("Float32", "FLOAT"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: allBoolean{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Bool", "BOOLEAN"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: &allBoolean{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Bool", "BOOLEAN"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: allTime{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Timestamp", "TIMESTAMP"),
|
|
||||||
reqField("Time", "TIME"),
|
|
||||||
reqField("Date", "DATE"),
|
|
||||||
reqField("DateTime", "DATETIME"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: &allNumeric{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("Numeric", "NUMERIC"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: allStrings{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("String", "STRING"),
|
|
||||||
reqField("ByteSlice", "BYTES"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range testCases {
|
|
||||||
got, err := InferSchema(tc.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, tc.want) {
|
|
||||||
t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in,
|
|
||||||
pretty.Value(got), pretty.Value(tc.want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type containsNested struct {
|
|
||||||
hidden string
|
|
||||||
NotNested int
|
|
||||||
Nested struct {
|
|
||||||
Inside int
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type containsDoubleNested struct {
|
|
||||||
NotNested int
|
|
||||||
Nested struct {
|
|
||||||
InsideNested struct {
|
|
||||||
Inside int
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ptrNested struct {
|
|
||||||
Ptr *struct{ Inside int }
|
|
||||||
}
|
|
||||||
|
|
||||||
type dup struct { // more than one field of the same struct type
|
|
||||||
A, B allBoolean
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNestedInference(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
want Schema
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: containsNested{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("NotNested", "INTEGER"),
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "Nested",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: containsDoubleNested{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("NotNested", "INTEGER"),
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "Nested",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{
|
|
||||||
{
|
|
||||||
Name: "InsideNested",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: ptrNested{},
|
|
||||||
want: Schema{
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "Ptr",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: dup{},
|
|
||||||
want: Schema{
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "A",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Bool", "BOOLEAN")},
|
|
||||||
},
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "B",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Bool", "BOOLEAN")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
got, err := InferSchema(tc.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, tc.want) {
|
|
||||||
t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in,
|
|
||||||
pretty.Value(got), pretty.Value(tc.want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type repeated struct {
|
|
||||||
NotRepeated []byte
|
|
||||||
RepeatedByteSlice [][]byte
|
|
||||||
Slice []int
|
|
||||||
Array [5]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type nestedRepeated struct {
|
|
||||||
NotRepeated int
|
|
||||||
Repeated []struct {
|
|
||||||
Inside int
|
|
||||||
}
|
|
||||||
RepeatedPtr []*struct{ Inside int }
|
|
||||||
}
|
|
||||||
|
|
||||||
func repField(name, typ string) *FieldSchema {
|
|
||||||
return &FieldSchema{
|
|
||||||
Name: name,
|
|
||||||
Type: FieldType(typ),
|
|
||||||
Repeated: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedInference(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
want Schema
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: repeated{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("NotRepeated", "BYTES"),
|
|
||||||
repField("RepeatedByteSlice", "BYTES"),
|
|
||||||
repField("Slice", "INTEGER"),
|
|
||||||
repField("Array", "BOOLEAN"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: nestedRepeated{},
|
|
||||||
want: Schema{
|
|
||||||
reqField("NotRepeated", "INTEGER"),
|
|
||||||
{
|
|
||||||
Name: "Repeated",
|
|
||||||
Repeated: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "RepeatedPtr",
|
|
||||||
Repeated: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("Inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, tc := range testCases {
|
|
||||||
got, err := InferSchema(tc.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%d: error inferring TableSchema: %v", i, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, tc.want) {
|
|
||||||
t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
|
|
||||||
pretty.Value(got), pretty.Value(tc.want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type allNulls struct {
|
|
||||||
A NullInt64
|
|
||||||
B NullFloat64
|
|
||||||
C NullBool
|
|
||||||
D NullString
|
|
||||||
E NullTimestamp
|
|
||||||
F NullTime
|
|
||||||
G NullDate
|
|
||||||
H NullDateTime
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNullInference(t *testing.T) {
|
|
||||||
got, err := InferSchema(allNulls{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := Schema{
|
|
||||||
optField("A", "INTEGER"),
|
|
||||||
optField("B", "FLOAT"),
|
|
||||||
optField("C", "BOOLEAN"),
|
|
||||||
optField("D", "STRING"),
|
|
||||||
optField("E", "TIMESTAMP"),
|
|
||||||
optField("F", "TIME"),
|
|
||||||
optField("G", "DATE"),
|
|
||||||
optField("H", "DATETIME"),
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, want); diff != "" {
|
|
||||||
t.Error(diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type Embedded struct {
|
|
||||||
Embedded int
|
|
||||||
}
|
|
||||||
|
|
||||||
type embedded struct {
|
|
||||||
Embedded2 int
|
|
||||||
}
|
|
||||||
|
|
||||||
type nestedEmbedded struct {
|
|
||||||
Embedded
|
|
||||||
embedded
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmbeddedInference(t *testing.T) {
|
|
||||||
got, err := InferSchema(nestedEmbedded{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := Schema{
|
|
||||||
reqField("Embedded", "INTEGER"),
|
|
||||||
reqField("Embedded2", "INTEGER"),
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, want) {
|
|
||||||
t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRecursiveInference(t *testing.T) {
|
|
||||||
type List struct {
|
|
||||||
Val int
|
|
||||||
Next *List
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := InferSchema(List{})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type withTags struct {
|
|
||||||
NoTag int
|
|
||||||
ExcludeTag int `bigquery:"-"`
|
|
||||||
SimpleTag int `bigquery:"simple_tag"`
|
|
||||||
UnderscoreTag int `bigquery:"_id"`
|
|
||||||
MixedCase int `bigquery:"MIXEDcase"`
|
|
||||||
Nullable []byte `bigquery:",nullable"`
|
|
||||||
NullNumeric *big.Rat `bigquery:",nullable"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type withTagsNested struct {
|
|
||||||
Nested withTags `bigquery:"nested"`
|
|
||||||
NestedAnonymous struct {
|
|
||||||
ExcludeTag int `bigquery:"-"`
|
|
||||||
Inside int `bigquery:"inside"`
|
|
||||||
} `bigquery:"anon"`
|
|
||||||
PNested *struct{ X int } // not nullable, for backwards compatibility
|
|
||||||
PNestedNullable *struct{ X int } `bigquery:",nullable"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type withTagsRepeated struct {
|
|
||||||
Repeated []withTags `bigquery:"repeated"`
|
|
||||||
RepeatedAnonymous []struct {
|
|
||||||
ExcludeTag int `bigquery:"-"`
|
|
||||||
Inside int `bigquery:"inside"`
|
|
||||||
} `bigquery:"anon"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type withTagsEmbedded struct {
|
|
||||||
withTags
|
|
||||||
}
|
|
||||||
|
|
||||||
var withTagsSchema = Schema{
|
|
||||||
reqField("NoTag", "INTEGER"),
|
|
||||||
reqField("simple_tag", "INTEGER"),
|
|
||||||
reqField("_id", "INTEGER"),
|
|
||||||
reqField("MIXEDcase", "INTEGER"),
|
|
||||||
optField("Nullable", "BYTES"),
|
|
||||||
optField("NullNumeric", "NUMERIC"),
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTagInference(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
want Schema
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: withTags{},
|
|
||||||
want: withTagsSchema,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: withTagsNested{},
|
|
||||||
want: Schema{
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "nested",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: withTagsSchema,
|
|
||||||
},
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "anon",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "PNested",
|
|
||||||
Required: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("X", "INTEGER")},
|
|
||||||
},
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "PNestedNullable",
|
|
||||||
Required: false,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("X", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: withTagsRepeated{},
|
|
||||||
want: Schema{
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "repeated",
|
|
||||||
Repeated: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: withTagsSchema,
|
|
||||||
},
|
|
||||||
&FieldSchema{
|
|
||||||
Name: "anon",
|
|
||||||
Repeated: true,
|
|
||||||
Type: "RECORD",
|
|
||||||
Schema: Schema{reqField("inside", "INTEGER")},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: withTagsEmbedded{},
|
|
||||||
want: withTagsSchema,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for i, tc := range testCases {
|
|
||||||
got, err := InferSchema(tc.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%d: error inferring TableSchema: %v", i, err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, tc.want) {
|
|
||||||
t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
|
|
||||||
pretty.Value(got), pretty.Value(tc.want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTagInferenceErrors(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
UnsupporedStartChar int `bigquery:"øab"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
UnsupportedEndChar int `bigquery:"abø"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
UnsupportedMiddleChar int `bigquery:"aøb"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
StartInt int `bigquery:"1abc"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
Hyphens int `bigquery:"a-b"`
|
|
||||||
}{},
|
|
||||||
err: errInvalidFieldName,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for i, tc := range testCases {
|
|
||||||
want := tc.err
|
|
||||||
_, got := InferSchema(tc.in)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := InferSchema(struct {
|
|
||||||
X int `bigquery:",optional"`
|
|
||||||
}{})
|
|
||||||
if err == nil {
|
|
||||||
t.Error("got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSchemaErrors(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
in interface{}
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: []byte{},
|
|
||||||
err: errNoStruct,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: new(int),
|
|
||||||
err: errNoStruct,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Uint uint }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Uint64 uint64 }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Uintptr uintptr }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Complex complex64 }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Map map[string]int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Chan chan bool }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Ptr *int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ Interface interface{} }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ MultiDimensional [][]int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ MultiDimensional [][][]byte }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ SliceOfPointer []*int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ SliceOfNull []NullInt64 }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ ChanSlice []chan bool }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ NestedChan struct{ Chan []chan bool } }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
X int `bigquery:",nullable"`
|
|
||||||
}{},
|
|
||||||
err: errBadNullable,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
X bool `bigquery:",nullable"`
|
|
||||||
}{},
|
|
||||||
err: errBadNullable,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
X struct{ N int } `bigquery:",nullable"`
|
|
||||||
}{},
|
|
||||||
err: errBadNullable,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct {
|
|
||||||
X []int `bigquery:",nullable"`
|
|
||||||
}{},
|
|
||||||
err: errBadNullable,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ X *[]byte }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ X *[]int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: struct{ X *int }{},
|
|
||||||
err: errUnsupportedFieldType,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range testCases {
|
|
||||||
want := tc.err
|
|
||||||
_, got := InferSchema(tc.in)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("%#v: got:\n%#v\nwant:\n%#v", tc.in, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHasRecursiveType(t *testing.T) {
|
|
||||||
type (
|
|
||||||
nonStruct int
|
|
||||||
nonRec struct{ A string }
|
|
||||||
dup struct{ A, B nonRec }
|
|
||||||
rec struct {
|
|
||||||
A int
|
|
||||||
B *rec
|
|
||||||
}
|
|
||||||
recUnexported struct {
|
|
||||||
A int
|
|
||||||
b *rec
|
|
||||||
}
|
|
||||||
hasRec struct {
|
|
||||||
A int
|
|
||||||
R *rec
|
|
||||||
}
|
|
||||||
recSlicePointer struct {
|
|
||||||
A []*recSlicePointer
|
|
||||||
}
|
|
||||||
)
|
|
||||||
for _, test := range []struct {
|
|
||||||
in interface{}
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{nonStruct(0), false},
|
|
||||||
{nonRec{}, false},
|
|
||||||
{dup{}, false},
|
|
||||||
{rec{}, true},
|
|
||||||
{recUnexported{}, false},
|
|
||||||
{hasRec{}, true},
|
|
||||||
{&recSlicePointer{}, true},
|
|
||||||
} {
|
|
||||||
got, err := hasRecursiveType(reflect.TypeOf(test.in), nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got != test.want {
|
|
||||||
t.Errorf("%T: got %t, want %t", test.in, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
531
vendor/cloud.google.com/go/bigquery/table.go
generated
vendored
531
vendor/cloud.google.com/go/bigquery/table.go
generated
vendored
@ -1,531 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/optional"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A Table is a reference to a BigQuery table.
|
|
||||||
type Table struct {
|
|
||||||
// ProjectID, DatasetID and TableID may be omitted if the Table is the destination for a query.
|
|
||||||
// In this case the result will be stored in an ephemeral table.
|
|
||||||
ProjectID string
|
|
||||||
DatasetID string
|
|
||||||
// TableID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_).
|
|
||||||
// The maximum length is 1,024 characters.
|
|
||||||
TableID string
|
|
||||||
|
|
||||||
c *Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableMetadata contains information about a BigQuery table.
|
|
||||||
type TableMetadata struct {
|
|
||||||
// The following fields can be set when creating a table.
|
|
||||||
|
|
||||||
// The user-friendly name for the table.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// The user-friendly description of the table.
|
|
||||||
Description string
|
|
||||||
|
|
||||||
// The table schema. If provided on create, ViewQuery must be empty.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// The query to use for a view. If provided on create, Schema must be nil.
|
|
||||||
ViewQuery string
|
|
||||||
|
|
||||||
// Use Legacy SQL for the view query.
|
|
||||||
// At most one of UseLegacySQL and UseStandardSQL can be true.
|
|
||||||
UseLegacySQL bool
|
|
||||||
|
|
||||||
// Use Legacy SQL for the view query. The default.
|
|
||||||
// At most one of UseLegacySQL and UseStandardSQL can be true.
|
|
||||||
// Deprecated: use UseLegacySQL.
|
|
||||||
UseStandardSQL bool
|
|
||||||
|
|
||||||
// If non-nil, the table is partitioned by time.
|
|
||||||
TimePartitioning *TimePartitioning
|
|
||||||
|
|
||||||
// The time when this table expires. If not set, the table will persist
|
|
||||||
// indefinitely. Expired tables will be deleted and their storage reclaimed.
|
|
||||||
ExpirationTime time.Time
|
|
||||||
|
|
||||||
// User-provided labels.
|
|
||||||
Labels map[string]string
|
|
||||||
|
|
||||||
// Information about a table stored outside of BigQuery.
|
|
||||||
ExternalDataConfig *ExternalDataConfig
|
|
||||||
|
|
||||||
// Custom encryption configuration (e.g., Cloud KMS keys).
|
|
||||||
EncryptionConfig *EncryptionConfig
|
|
||||||
|
|
||||||
// All the fields below are read-only.
|
|
||||||
|
|
||||||
FullID string // An opaque ID uniquely identifying the table.
|
|
||||||
Type TableType
|
|
||||||
CreationTime time.Time
|
|
||||||
LastModifiedTime time.Time
|
|
||||||
|
|
||||||
// The size of the table in bytes.
|
|
||||||
// This does not include data that is being buffered during a streaming insert.
|
|
||||||
NumBytes int64
|
|
||||||
|
|
||||||
// The number of rows of data in this table.
|
|
||||||
// This does not include data that is being buffered during a streaming insert.
|
|
||||||
NumRows uint64
|
|
||||||
|
|
||||||
// Contains information regarding this table's streaming buffer, if one is
|
|
||||||
// present. This field will be nil if the table is not being streamed to or if
|
|
||||||
// there is no data in the streaming buffer.
|
|
||||||
StreamingBuffer *StreamingBuffer
|
|
||||||
|
|
||||||
// ETag is the ETag obtained when reading metadata. Pass it to Table.Update to
|
|
||||||
// ensure that the metadata hasn't changed since it was read.
|
|
||||||
ETag string
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableCreateDisposition specifies the circumstances under which destination table will be created.
|
|
||||||
// Default is CreateIfNeeded.
|
|
||||||
type TableCreateDisposition string
|
|
||||||
|
|
||||||
const (
|
|
||||||
// CreateIfNeeded will create the table if it does not already exist.
|
|
||||||
// Tables are created atomically on successful completion of a job.
|
|
||||||
CreateIfNeeded TableCreateDisposition = "CREATE_IF_NEEDED"
|
|
||||||
|
|
||||||
// CreateNever ensures the table must already exist and will not be
|
|
||||||
// automatically created.
|
|
||||||
CreateNever TableCreateDisposition = "CREATE_NEVER"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TableWriteDisposition specifies how existing data in a destination table is treated.
|
|
||||||
// Default is WriteAppend.
|
|
||||||
type TableWriteDisposition string
|
|
||||||
|
|
||||||
const (
|
|
||||||
// WriteAppend will append to any existing data in the destination table.
|
|
||||||
// Data is appended atomically on successful completion of a job.
|
|
||||||
WriteAppend TableWriteDisposition = "WRITE_APPEND"
|
|
||||||
|
|
||||||
// WriteTruncate overrides the existing data in the destination table.
|
|
||||||
// Data is overwritten atomically on successful completion of a job.
|
|
||||||
WriteTruncate TableWriteDisposition = "WRITE_TRUNCATE"
|
|
||||||
|
|
||||||
// WriteEmpty fails writes if the destination table already contains data.
|
|
||||||
WriteEmpty TableWriteDisposition = "WRITE_EMPTY"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TableType is the type of table.
|
|
||||||
type TableType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
RegularTable TableType = "TABLE"
|
|
||||||
ViewTable TableType = "VIEW"
|
|
||||||
ExternalTable TableType = "EXTERNAL"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TimePartitioning describes the time-based date partitioning on a table.
|
|
||||||
// For more information see: https://cloud.google.com/bigquery/docs/creating-partitioned-tables.
|
|
||||||
type TimePartitioning struct {
|
|
||||||
// The amount of time to keep the storage for a partition.
|
|
||||||
// If the duration is empty (0), the data in the partitions do not expire.
|
|
||||||
Expiration time.Duration
|
|
||||||
|
|
||||||
// If empty, the table is partitioned by pseudo column '_PARTITIONTIME'; if set, the
|
|
||||||
// table is partitioned by this field. The field must be a top-level TIMESTAMP or
|
|
||||||
// DATE field. Its mode must be NULLABLE or REQUIRED.
|
|
||||||
Field string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *TimePartitioning) toBQ() *bq.TimePartitioning {
|
|
||||||
if p == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &bq.TimePartitioning{
|
|
||||||
Type: "DAY",
|
|
||||||
ExpirationMs: int64(p.Expiration / time.Millisecond),
|
|
||||||
Field: p.Field,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToTimePartitioning(q *bq.TimePartitioning) *TimePartitioning {
|
|
||||||
if q == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &TimePartitioning{
|
|
||||||
Expiration: time.Duration(q.ExpirationMs) * time.Millisecond,
|
|
||||||
Field: q.Field,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// EncryptionConfig configures customer-managed encryption on tables.
|
|
||||||
type EncryptionConfig struct {
|
|
||||||
// Describes the Cloud KMS encryption key that will be used to protect
|
|
||||||
// destination BigQuery table. The BigQuery Service Account associated with your
|
|
||||||
// project requires access to this encryption key.
|
|
||||||
KMSKeyName string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *EncryptionConfig) toBQ() *bq.EncryptionConfiguration {
|
|
||||||
if e == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &bq.EncryptionConfiguration{
|
|
||||||
KmsKeyName: e.KMSKeyName,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToEncryptionConfig(q *bq.EncryptionConfiguration) *EncryptionConfig {
|
|
||||||
if q == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &EncryptionConfig{
|
|
||||||
KMSKeyName: q.KmsKeyName,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// StreamingBuffer holds information about the streaming buffer.
|
|
||||||
type StreamingBuffer struct {
|
|
||||||
// A lower-bound estimate of the number of bytes currently in the streaming
|
|
||||||
// buffer.
|
|
||||||
EstimatedBytes uint64
|
|
||||||
|
|
||||||
// A lower-bound estimate of the number of rows currently in the streaming
|
|
||||||
// buffer.
|
|
||||||
EstimatedRows uint64
|
|
||||||
|
|
||||||
// The time of the oldest entry in the streaming buffer.
|
|
||||||
OldestEntryTime time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *Table) toBQ() *bq.TableReference {
|
|
||||||
return &bq.TableReference{
|
|
||||||
ProjectId: t.ProjectID,
|
|
||||||
DatasetId: t.DatasetID,
|
|
||||||
TableId: t.TableID,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FullyQualifiedName returns the ID of the table in projectID:datasetID.tableID format.
|
|
||||||
func (t *Table) FullyQualifiedName() string {
|
|
||||||
return fmt.Sprintf("%s:%s.%s", t.ProjectID, t.DatasetID, t.TableID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// implicitTable reports whether Table is an empty placeholder, which signifies that a new table should be created with an auto-generated Table ID.
|
|
||||||
func (t *Table) implicitTable() bool {
|
|
||||||
return t.ProjectID == "" && t.DatasetID == "" && t.TableID == ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create creates a table in the BigQuery service.
|
|
||||||
// Pass in a TableMetadata value to configure the table.
|
|
||||||
// If tm.View.Query is non-empty, the created table will be of type VIEW.
|
|
||||||
// Expiration can only be set during table creation.
|
|
||||||
// After table creation, a view can be modified only if its table was initially created
|
|
||||||
// with a view.
|
|
||||||
func (t *Table) Create(ctx context.Context, tm *TableMetadata) (err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Table.Create")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
table, err := tm.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
table.TableReference = &bq.TableReference{
|
|
||||||
ProjectId: t.ProjectID,
|
|
||||||
DatasetId: t.DatasetID,
|
|
||||||
TableId: t.TableID,
|
|
||||||
}
|
|
||||||
req := t.c.bqs.Tables.Insert(t.ProjectID, t.DatasetID, table).Context(ctx)
|
|
||||||
setClientHeader(req.Header())
|
|
||||||
_, err = req.Do()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tm *TableMetadata) toBQ() (*bq.Table, error) {
|
|
||||||
t := &bq.Table{}
|
|
||||||
if tm == nil {
|
|
||||||
return t, nil
|
|
||||||
}
|
|
||||||
if tm.Schema != nil && tm.ViewQuery != "" {
|
|
||||||
return nil, errors.New("bigquery: provide Schema or ViewQuery, not both")
|
|
||||||
}
|
|
||||||
t.FriendlyName = tm.Name
|
|
||||||
t.Description = tm.Description
|
|
||||||
t.Labels = tm.Labels
|
|
||||||
if tm.Schema != nil {
|
|
||||||
t.Schema = tm.Schema.toBQ()
|
|
||||||
}
|
|
||||||
if tm.ViewQuery != "" {
|
|
||||||
if tm.UseStandardSQL && tm.UseLegacySQL {
|
|
||||||
return nil, errors.New("bigquery: cannot provide both UseStandardSQL and UseLegacySQL")
|
|
||||||
}
|
|
||||||
t.View = &bq.ViewDefinition{Query: tm.ViewQuery}
|
|
||||||
if tm.UseLegacySQL {
|
|
||||||
t.View.UseLegacySql = true
|
|
||||||
} else {
|
|
||||||
t.View.UseLegacySql = false
|
|
||||||
t.View.ForceSendFields = append(t.View.ForceSendFields, "UseLegacySql")
|
|
||||||
}
|
|
||||||
} else if tm.UseLegacySQL || tm.UseStandardSQL {
|
|
||||||
return nil, errors.New("bigquery: UseLegacy/StandardSQL requires ViewQuery")
|
|
||||||
}
|
|
||||||
t.TimePartitioning = tm.TimePartitioning.toBQ()
|
|
||||||
if !tm.ExpirationTime.IsZero() {
|
|
||||||
t.ExpirationTime = tm.ExpirationTime.UnixNano() / 1e6
|
|
||||||
}
|
|
||||||
if tm.ExternalDataConfig != nil {
|
|
||||||
edc := tm.ExternalDataConfig.toBQ()
|
|
||||||
t.ExternalDataConfiguration = &edc
|
|
||||||
}
|
|
||||||
t.EncryptionConfiguration = tm.EncryptionConfig.toBQ()
|
|
||||||
if tm.FullID != "" {
|
|
||||||
return nil, errors.New("cannot set FullID on create")
|
|
||||||
}
|
|
||||||
if tm.Type != "" {
|
|
||||||
return nil, errors.New("cannot set Type on create")
|
|
||||||
}
|
|
||||||
if !tm.CreationTime.IsZero() {
|
|
||||||
return nil, errors.New("cannot set CreationTime on create")
|
|
||||||
}
|
|
||||||
if !tm.LastModifiedTime.IsZero() {
|
|
||||||
return nil, errors.New("cannot set LastModifiedTime on create")
|
|
||||||
}
|
|
||||||
if tm.NumBytes != 0 {
|
|
||||||
return nil, errors.New("cannot set NumBytes on create")
|
|
||||||
}
|
|
||||||
if tm.NumRows != 0 {
|
|
||||||
return nil, errors.New("cannot set NumRows on create")
|
|
||||||
}
|
|
||||||
if tm.StreamingBuffer != nil {
|
|
||||||
return nil, errors.New("cannot set StreamingBuffer on create")
|
|
||||||
}
|
|
||||||
if tm.ETag != "" {
|
|
||||||
return nil, errors.New("cannot set ETag on create")
|
|
||||||
}
|
|
||||||
return t, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Metadata fetches the metadata for the table.
|
|
||||||
func (t *Table) Metadata(ctx context.Context) (md *TableMetadata, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Table.Metadata")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
req := t.c.bqs.Tables.Get(t.ProjectID, t.DatasetID, t.TableID).Context(ctx)
|
|
||||||
setClientHeader(req.Header())
|
|
||||||
var table *bq.Table
|
|
||||||
err = runWithRetry(ctx, func() (err error) {
|
|
||||||
table, err = req.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToTableMetadata(table)
|
|
||||||
}
|
|
||||||
|
|
||||||
func bqToTableMetadata(t *bq.Table) (*TableMetadata, error) {
|
|
||||||
md := &TableMetadata{
|
|
||||||
Description: t.Description,
|
|
||||||
Name: t.FriendlyName,
|
|
||||||
Type: TableType(t.Type),
|
|
||||||
FullID: t.Id,
|
|
||||||
Labels: t.Labels,
|
|
||||||
NumBytes: t.NumBytes,
|
|
||||||
NumRows: t.NumRows,
|
|
||||||
ExpirationTime: unixMillisToTime(t.ExpirationTime),
|
|
||||||
CreationTime: unixMillisToTime(t.CreationTime),
|
|
||||||
LastModifiedTime: unixMillisToTime(int64(t.LastModifiedTime)),
|
|
||||||
ETag: t.Etag,
|
|
||||||
EncryptionConfig: bqToEncryptionConfig(t.EncryptionConfiguration),
|
|
||||||
}
|
|
||||||
if t.Schema != nil {
|
|
||||||
md.Schema = bqToSchema(t.Schema)
|
|
||||||
}
|
|
||||||
if t.View != nil {
|
|
||||||
md.ViewQuery = t.View.Query
|
|
||||||
md.UseLegacySQL = t.View.UseLegacySql
|
|
||||||
}
|
|
||||||
md.TimePartitioning = bqToTimePartitioning(t.TimePartitioning)
|
|
||||||
if t.StreamingBuffer != nil {
|
|
||||||
md.StreamingBuffer = &StreamingBuffer{
|
|
||||||
EstimatedBytes: t.StreamingBuffer.EstimatedBytes,
|
|
||||||
EstimatedRows: t.StreamingBuffer.EstimatedRows,
|
|
||||||
OldestEntryTime: unixMillisToTime(int64(t.StreamingBuffer.OldestEntryTime)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if t.ExternalDataConfiguration != nil {
|
|
||||||
edc, err := bqToExternalDataConfig(t.ExternalDataConfiguration)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
md.ExternalDataConfig = edc
|
|
||||||
}
|
|
||||||
return md, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete deletes the table.
|
|
||||||
func (t *Table) Delete(ctx context.Context) (err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Table.Delete")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
req := t.c.bqs.Tables.Delete(t.ProjectID, t.DatasetID, t.TableID).Context(ctx)
|
|
||||||
setClientHeader(req.Header())
|
|
||||||
return req.Do()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read fetches the contents of the table.
|
|
||||||
func (t *Table) Read(ctx context.Context) *RowIterator {
|
|
||||||
return t.read(ctx, fetchPage)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *Table) read(ctx context.Context, pf pageFetcher) *RowIterator {
|
|
||||||
return newRowIterator(ctx, t, pf)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update modifies specific Table metadata fields.
|
|
||||||
func (t *Table) Update(ctx context.Context, tm TableMetadataToUpdate, etag string) (md *TableMetadata, err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Table.Update")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
bqt := tm.toBQ()
|
|
||||||
call := t.c.bqs.Tables.Patch(t.ProjectID, t.DatasetID, t.TableID, bqt).Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
if etag != "" {
|
|
||||||
call.Header().Set("If-Match", etag)
|
|
||||||
}
|
|
||||||
var res *bq.Table
|
|
||||||
if err := runWithRetry(ctx, func() (err error) {
|
|
||||||
res, err = call.Do()
|
|
||||||
return err
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return bqToTableMetadata(res)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tm *TableMetadataToUpdate) toBQ() *bq.Table {
|
|
||||||
t := &bq.Table{}
|
|
||||||
forceSend := func(field string) {
|
|
||||||
t.ForceSendFields = append(t.ForceSendFields, field)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tm.Description != nil {
|
|
||||||
t.Description = optional.ToString(tm.Description)
|
|
||||||
forceSend("Description")
|
|
||||||
}
|
|
||||||
if tm.Name != nil {
|
|
||||||
t.FriendlyName = optional.ToString(tm.Name)
|
|
||||||
forceSend("FriendlyName")
|
|
||||||
}
|
|
||||||
if tm.Schema != nil {
|
|
||||||
t.Schema = tm.Schema.toBQ()
|
|
||||||
forceSend("Schema")
|
|
||||||
}
|
|
||||||
if !tm.ExpirationTime.IsZero() {
|
|
||||||
t.ExpirationTime = tm.ExpirationTime.UnixNano() / 1e6
|
|
||||||
forceSend("ExpirationTime")
|
|
||||||
}
|
|
||||||
if tm.ViewQuery != nil {
|
|
||||||
t.View = &bq.ViewDefinition{
|
|
||||||
Query: optional.ToString(tm.ViewQuery),
|
|
||||||
ForceSendFields: []string{"Query"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if tm.UseLegacySQL != nil {
|
|
||||||
if t.View == nil {
|
|
||||||
t.View = &bq.ViewDefinition{}
|
|
||||||
}
|
|
||||||
t.View.UseLegacySql = optional.ToBool(tm.UseLegacySQL)
|
|
||||||
t.View.ForceSendFields = append(t.View.ForceSendFields, "UseLegacySql")
|
|
||||||
}
|
|
||||||
labels, forces, nulls := tm.update()
|
|
||||||
t.Labels = labels
|
|
||||||
t.ForceSendFields = append(t.ForceSendFields, forces...)
|
|
||||||
t.NullFields = append(t.NullFields, nulls...)
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// TableMetadataToUpdate is used when updating a table's metadata.
|
|
||||||
// Only non-nil fields will be updated.
|
|
||||||
type TableMetadataToUpdate struct {
|
|
||||||
// The user-friendly description of this table.
|
|
||||||
Description optional.String
|
|
||||||
|
|
||||||
// The user-friendly name for this table.
|
|
||||||
Name optional.String
|
|
||||||
|
|
||||||
// The table's schema.
|
|
||||||
// When updating a schema, you can add columns but not remove them.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// The time when this table expires.
|
|
||||||
ExpirationTime time.Time
|
|
||||||
|
|
||||||
// The query to use for a view.
|
|
||||||
ViewQuery optional.String
|
|
||||||
|
|
||||||
// Use Legacy SQL for the view query.
|
|
||||||
UseLegacySQL optional.Bool
|
|
||||||
|
|
||||||
labelUpdater
|
|
||||||
}
|
|
||||||
|
|
||||||
// labelUpdater contains common code for updating labels.
|
|
||||||
type labelUpdater struct {
|
|
||||||
setLabels map[string]string
|
|
||||||
deleteLabels map[string]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetLabel causes a label to be added or modified on a call to Update.
|
|
||||||
func (u *labelUpdater) SetLabel(name, value string) {
|
|
||||||
if u.setLabels == nil {
|
|
||||||
u.setLabels = map[string]string{}
|
|
||||||
}
|
|
||||||
u.setLabels[name] = value
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteLabel causes a label to be deleted on a call to Update.
|
|
||||||
func (u *labelUpdater) DeleteLabel(name string) {
|
|
||||||
if u.deleteLabels == nil {
|
|
||||||
u.deleteLabels = map[string]bool{}
|
|
||||||
}
|
|
||||||
u.deleteLabels[name] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *labelUpdater) update() (labels map[string]string, forces, nulls []string) {
|
|
||||||
if u.setLabels == nil && u.deleteLabels == nil {
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
labels = map[string]string{}
|
|
||||||
for k, v := range u.setLabels {
|
|
||||||
labels[k] = v
|
|
||||||
}
|
|
||||||
if len(labels) == 0 && len(u.deleteLabels) > 0 {
|
|
||||||
forces = []string{"Labels"}
|
|
||||||
}
|
|
||||||
for l := range u.deleteLabels {
|
|
||||||
nulls = append(nulls, "Labels."+l)
|
|
||||||
}
|
|
||||||
return labels, forces, nulls
|
|
||||||
}
|
|
||||||
295
vendor/cloud.google.com/go/bigquery/table_test.go
generated
vendored
295
vendor/cloud.google.com/go/bigquery/table_test.go
generated
vendored
@ -1,295 +0,0 @@
|
|||||||
// Copyright 2017 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBQToTableMetadata(t *testing.T) {
|
|
||||||
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
||||||
aTimeMillis := aTime.UnixNano() / 1e6
|
|
||||||
for _, test := range []struct {
|
|
||||||
in *bq.Table
|
|
||||||
want *TableMetadata
|
|
||||||
}{
|
|
||||||
{&bq.Table{}, &TableMetadata{}}, // test minimal case
|
|
||||||
{
|
|
||||||
&bq.Table{
|
|
||||||
CreationTime: aTimeMillis,
|
|
||||||
Description: "desc",
|
|
||||||
Etag: "etag",
|
|
||||||
ExpirationTime: aTimeMillis,
|
|
||||||
FriendlyName: "fname",
|
|
||||||
Id: "id",
|
|
||||||
LastModifiedTime: uint64(aTimeMillis),
|
|
||||||
Location: "loc",
|
|
||||||
NumBytes: 123,
|
|
||||||
NumLongTermBytes: 23,
|
|
||||||
NumRows: 7,
|
|
||||||
StreamingBuffer: &bq.Streamingbuffer{
|
|
||||||
EstimatedBytes: 11,
|
|
||||||
EstimatedRows: 3,
|
|
||||||
OldestEntryTime: uint64(aTimeMillis),
|
|
||||||
},
|
|
||||||
TimePartitioning: &bq.TimePartitioning{
|
|
||||||
ExpirationMs: 7890,
|
|
||||||
Type: "DAY",
|
|
||||||
Field: "pfield",
|
|
||||||
},
|
|
||||||
EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
|
|
||||||
Type: "EXTERNAL",
|
|
||||||
View: &bq.ViewDefinition{Query: "view-query"},
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
ExternalDataConfiguration: &bq.ExternalDataConfiguration{
|
|
||||||
SourceFormat: "GOOGLE_SHEETS",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
&TableMetadata{
|
|
||||||
Description: "desc",
|
|
||||||
Name: "fname",
|
|
||||||
ViewQuery: "view-query",
|
|
||||||
FullID: "id",
|
|
||||||
Type: ExternalTable,
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets},
|
|
||||||
ExpirationTime: aTime.Truncate(time.Millisecond),
|
|
||||||
CreationTime: aTime.Truncate(time.Millisecond),
|
|
||||||
LastModifiedTime: aTime.Truncate(time.Millisecond),
|
|
||||||
NumBytes: 123,
|
|
||||||
NumRows: 7,
|
|
||||||
TimePartitioning: &TimePartitioning{
|
|
||||||
Expiration: 7890 * time.Millisecond,
|
|
||||||
Field: "pfield",
|
|
||||||
},
|
|
||||||
StreamingBuffer: &StreamingBuffer{
|
|
||||||
EstimatedBytes: 11,
|
|
||||||
EstimatedRows: 3,
|
|
||||||
OldestEntryTime: aTime,
|
|
||||||
},
|
|
||||||
EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
||||||
ETag: "etag",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
got, err := bqToTableMetadata(test.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, test.want); diff != "" {
|
|
||||||
t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTableMetadataToBQ(t *testing.T) {
|
|
||||||
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
||||||
aTimeMillis := aTime.UnixNano() / 1e6
|
|
||||||
sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
|
|
||||||
|
|
||||||
for _, test := range []struct {
|
|
||||||
in *TableMetadata
|
|
||||||
want *bq.Table
|
|
||||||
}{
|
|
||||||
{nil, &bq.Table{}},
|
|
||||||
{&TableMetadata{}, &bq.Table{}},
|
|
||||||
{
|
|
||||||
&TableMetadata{
|
|
||||||
Name: "n",
|
|
||||||
Description: "d",
|
|
||||||
Schema: sc,
|
|
||||||
ExpirationTime: aTime,
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
|
|
||||||
EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
|
|
||||||
},
|
|
||||||
&bq.Table{
|
|
||||||
FriendlyName: "n",
|
|
||||||
Description: "d",
|
|
||||||
Schema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ExpirationTime: aTimeMillis,
|
|
||||||
Labels: map[string]string{"a": "b"},
|
|
||||||
ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
|
|
||||||
EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
&TableMetadata{ViewQuery: "q"},
|
|
||||||
&bq.Table{
|
|
||||||
View: &bq.ViewDefinition{
|
|
||||||
Query: "q",
|
|
||||||
UseLegacySql: false,
|
|
||||||
ForceSendFields: []string{"UseLegacySql"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
&TableMetadata{
|
|
||||||
ViewQuery: "q",
|
|
||||||
UseLegacySQL: true,
|
|
||||||
TimePartitioning: &TimePartitioning{},
|
|
||||||
},
|
|
||||||
&bq.Table{
|
|
||||||
View: &bq.ViewDefinition{
|
|
||||||
Query: "q",
|
|
||||||
UseLegacySql: true,
|
|
||||||
},
|
|
||||||
TimePartitioning: &bq.TimePartitioning{
|
|
||||||
Type: "DAY",
|
|
||||||
ExpirationMs: 0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
&TableMetadata{
|
|
||||||
ViewQuery: "q",
|
|
||||||
UseStandardSQL: true,
|
|
||||||
TimePartitioning: &TimePartitioning{
|
|
||||||
Expiration: time.Second,
|
|
||||||
Field: "ofDreams",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
&bq.Table{
|
|
||||||
View: &bq.ViewDefinition{
|
|
||||||
Query: "q",
|
|
||||||
UseLegacySql: false,
|
|
||||||
ForceSendFields: []string{"UseLegacySql"},
|
|
||||||
},
|
|
||||||
TimePartitioning: &bq.TimePartitioning{
|
|
||||||
Type: "DAY",
|
|
||||||
ExpirationMs: 1000,
|
|
||||||
Field: "ofDreams",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
got, err := test.in.toBQ()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v: %v", test.in, err)
|
|
||||||
}
|
|
||||||
if diff := testutil.Diff(got, test.want); diff != "" {
|
|
||||||
t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Errors
|
|
||||||
for _, in := range []*TableMetadata{
|
|
||||||
{Schema: sc, ViewQuery: "q"}, // can't have both schema and query
|
|
||||||
{UseLegacySQL: true}, // UseLegacySQL without query
|
|
||||||
{UseStandardSQL: true}, // UseStandardSQL without query
|
|
||||||
// read-only fields
|
|
||||||
{FullID: "x"},
|
|
||||||
{Type: "x"},
|
|
||||||
{CreationTime: aTime},
|
|
||||||
{LastModifiedTime: aTime},
|
|
||||||
{NumBytes: 1},
|
|
||||||
{NumRows: 1},
|
|
||||||
{StreamingBuffer: &StreamingBuffer{}},
|
|
||||||
{ETag: "x"},
|
|
||||||
} {
|
|
||||||
_, err := in.toBQ()
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("%+v: got nil, want error", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTableMetadataToUpdateToBQ(t *testing.T) {
|
|
||||||
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
|
|
||||||
for _, test := range []struct {
|
|
||||||
tm TableMetadataToUpdate
|
|
||||||
want *bq.Table
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{},
|
|
||||||
want: &bq.Table{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{
|
|
||||||
Description: "d",
|
|
||||||
Name: "n",
|
|
||||||
},
|
|
||||||
want: &bq.Table{
|
|
||||||
Description: "d",
|
|
||||||
FriendlyName: "n",
|
|
||||||
ForceSendFields: []string{"Description", "FriendlyName"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{
|
|
||||||
Schema: Schema{fieldSchema("desc", "name", "STRING", false, true)},
|
|
||||||
ExpirationTime: aTime,
|
|
||||||
},
|
|
||||||
want: &bq.Table{
|
|
||||||
Schema: &bq.TableSchema{
|
|
||||||
Fields: []*bq.TableFieldSchema{
|
|
||||||
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
ExpirationTime: aTime.UnixNano() / 1e6,
|
|
||||||
ForceSendFields: []string{"Schema", "ExpirationTime"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{ViewQuery: "q"},
|
|
||||||
want: &bq.Table{
|
|
||||||
View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{UseLegacySQL: false},
|
|
||||||
want: &bq.Table{
|
|
||||||
View: &bq.ViewDefinition{
|
|
||||||
UseLegacySql: false,
|
|
||||||
ForceSendFields: []string{"UseLegacySql"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true},
|
|
||||||
want: &bq.Table{
|
|
||||||
View: &bq.ViewDefinition{
|
|
||||||
Query: "q",
|
|
||||||
UseLegacySql: true,
|
|
||||||
ForceSendFields: []string{"Query", "UseLegacySql"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
tm: func() (tm TableMetadataToUpdate) {
|
|
||||||
tm.SetLabel("L", "V")
|
|
||||||
tm.DeleteLabel("D")
|
|
||||||
return tm
|
|
||||||
}(),
|
|
||||||
want: &bq.Table{
|
|
||||||
Labels: map[string]string{"L": "V"},
|
|
||||||
NullFields: []string{"Labels.D"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
got := test.tm.toBQ()
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%+v:\ngot %+v\nwant %+v", test.tm, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
231
vendor/cloud.google.com/go/bigquery/uploader.go
generated
vendored
231
vendor/cloud.google.com/go/bigquery/uploader.go
generated
vendored
@ -1,231 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/trace"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// An Uploader does streaming inserts into a BigQuery table.
|
|
||||||
// It is safe for concurrent use.
|
|
||||||
type Uploader struct {
|
|
||||||
t *Table
|
|
||||||
|
|
||||||
// SkipInvalidRows causes rows containing invalid data to be silently
|
|
||||||
// ignored. The default value is false, which causes the entire request to
|
|
||||||
// fail if there is an attempt to insert an invalid row.
|
|
||||||
SkipInvalidRows bool
|
|
||||||
|
|
||||||
// IgnoreUnknownValues causes values not matching the schema to be ignored.
|
|
||||||
// The default value is false, which causes records containing such values
|
|
||||||
// to be treated as invalid records.
|
|
||||||
IgnoreUnknownValues bool
|
|
||||||
|
|
||||||
// A TableTemplateSuffix allows Uploaders to create tables automatically.
|
|
||||||
//
|
|
||||||
// Experimental: this option is experimental and may be modified or removed in future versions,
|
|
||||||
// regardless of any other documented package stability guarantees.
|
|
||||||
//
|
|
||||||
// When you specify a suffix, the table you upload data to
|
|
||||||
// will be used as a template for creating a new table, with the same schema,
|
|
||||||
// called <table> + <suffix>.
|
|
||||||
//
|
|
||||||
// More information is available at
|
|
||||||
// https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
|
|
||||||
TableTemplateSuffix string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Uploader returns an Uploader that can be used to append rows to t.
|
|
||||||
// The returned Uploader may optionally be further configured before its Put method is called.
|
|
||||||
//
|
|
||||||
// To stream rows into a date-partitioned table at a particular date, add the
|
|
||||||
// $yyyymmdd suffix to the table name when constructing the Table.
|
|
||||||
func (t *Table) Uploader() *Uploader {
|
|
||||||
return &Uploader{t: t}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put uploads one or more rows to the BigQuery service.
|
|
||||||
//
|
|
||||||
// If src is ValueSaver, then its Save method is called to produce a row for uploading.
|
|
||||||
//
|
|
||||||
// If src is a struct or pointer to a struct, then a schema is inferred from it
|
|
||||||
// and used to create a StructSaver. The InsertID of the StructSaver will be
|
|
||||||
// empty.
|
|
||||||
//
|
|
||||||
// If src is a slice of ValueSavers, structs, or struct pointers, then each
|
|
||||||
// element of the slice is treated as above, and multiple rows are uploaded.
|
|
||||||
//
|
|
||||||
// Put returns a PutMultiError if one or more rows failed to be uploaded.
|
|
||||||
// The PutMultiError contains a RowInsertionError for each failed row.
|
|
||||||
//
|
|
||||||
// Put will retry on temporary errors (see
|
|
||||||
// https://cloud.google.com/bigquery/troubleshooting-errors). This can result
|
|
||||||
// in duplicate rows if you do not use insert IDs. Also, if the error persists,
|
|
||||||
// the call will run indefinitely. Pass a context with a timeout to prevent
|
|
||||||
// hanging calls.
|
|
||||||
func (u *Uploader) Put(ctx context.Context, src interface{}) (err error) {
|
|
||||||
ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Uploader.Put")
|
|
||||||
defer func() { trace.EndSpan(ctx, err) }()
|
|
||||||
|
|
||||||
savers, err := valueSavers(src)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return u.putMulti(ctx, savers)
|
|
||||||
}
|
|
||||||
|
|
||||||
func valueSavers(src interface{}) ([]ValueSaver, error) {
|
|
||||||
saver, ok, err := toValueSaver(src)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if ok {
|
|
||||||
return []ValueSaver{saver}, nil
|
|
||||||
}
|
|
||||||
srcVal := reflect.ValueOf(src)
|
|
||||||
if srcVal.Kind() != reflect.Slice {
|
|
||||||
return nil, fmt.Errorf("%T is not a ValueSaver, struct, struct pointer, or slice", src)
|
|
||||||
|
|
||||||
}
|
|
||||||
var savers []ValueSaver
|
|
||||||
for i := 0; i < srcVal.Len(); i++ {
|
|
||||||
s := srcVal.Index(i).Interface()
|
|
||||||
saver, ok, err := toValueSaver(s)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("src[%d] has type %T, which is not a ValueSaver, struct or struct pointer", i, s)
|
|
||||||
}
|
|
||||||
savers = append(savers, saver)
|
|
||||||
}
|
|
||||||
return savers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make a ValueSaver from x, which must implement ValueSaver already
|
|
||||||
// or be a struct or pointer to struct.
|
|
||||||
func toValueSaver(x interface{}) (ValueSaver, bool, error) {
|
|
||||||
if _, ok := x.(StructSaver); ok {
|
|
||||||
return nil, false, errors.New("bigquery: use &StructSaver, not StructSaver")
|
|
||||||
}
|
|
||||||
var insertID string
|
|
||||||
// Handle StructSavers specially so we can infer the schema if necessary.
|
|
||||||
if ss, ok := x.(*StructSaver); ok && ss.Schema == nil {
|
|
||||||
x = ss.Struct
|
|
||||||
insertID = ss.InsertID
|
|
||||||
// Fall through so we can infer the schema.
|
|
||||||
}
|
|
||||||
if saver, ok := x.(ValueSaver); ok {
|
|
||||||
return saver, ok, nil
|
|
||||||
}
|
|
||||||
v := reflect.ValueOf(x)
|
|
||||||
// Support Put with []interface{}
|
|
||||||
if v.Kind() == reflect.Interface {
|
|
||||||
v = v.Elem()
|
|
||||||
}
|
|
||||||
if v.Kind() == reflect.Ptr {
|
|
||||||
v = v.Elem()
|
|
||||||
}
|
|
||||||
if v.Kind() != reflect.Struct {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
schema, err := inferSchemaReflectCached(v.Type())
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
return &StructSaver{
|
|
||||||
Struct: x,
|
|
||||||
InsertID: insertID,
|
|
||||||
Schema: schema,
|
|
||||||
}, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
|
|
||||||
req, err := u.newInsertRequest(src)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if req == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
call := u.t.c.bqs.Tabledata.InsertAll(u.t.ProjectID, u.t.DatasetID, u.t.TableID, req)
|
|
||||||
call = call.Context(ctx)
|
|
||||||
setClientHeader(call.Header())
|
|
||||||
var res *bq.TableDataInsertAllResponse
|
|
||||||
err = runWithRetry(ctx, func() (err error) {
|
|
||||||
res, err = call.Do()
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return handleInsertErrors(res.InsertErrors, req.Rows)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *Uploader) newInsertRequest(savers []ValueSaver) (*bq.TableDataInsertAllRequest, error) {
|
|
||||||
if savers == nil { // If there are no rows, do nothing.
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
req := &bq.TableDataInsertAllRequest{
|
|
||||||
TemplateSuffix: u.TableTemplateSuffix,
|
|
||||||
IgnoreUnknownValues: u.IgnoreUnknownValues,
|
|
||||||
SkipInvalidRows: u.SkipInvalidRows,
|
|
||||||
}
|
|
||||||
for _, saver := range savers {
|
|
||||||
row, insertID, err := saver.Save()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if insertID == "" {
|
|
||||||
insertID = randomIDFn()
|
|
||||||
}
|
|
||||||
m := make(map[string]bq.JsonValue)
|
|
||||||
for k, v := range row {
|
|
||||||
m[k] = bq.JsonValue(v)
|
|
||||||
}
|
|
||||||
req.Rows = append(req.Rows, &bq.TableDataInsertAllRequestRows{
|
|
||||||
InsertId: insertID,
|
|
||||||
Json: m,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return req, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleInsertErrors(ierrs []*bq.TableDataInsertAllResponseInsertErrors, rows []*bq.TableDataInsertAllRequestRows) error {
|
|
||||||
if len(ierrs) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var errs PutMultiError
|
|
||||||
for _, e := range ierrs {
|
|
||||||
if int(e.Index) > len(rows) {
|
|
||||||
return fmt.Errorf("internal error: unexpected row index: %v", e.Index)
|
|
||||||
}
|
|
||||||
rie := RowInsertionError{
|
|
||||||
InsertID: rows[e.Index].InsertId,
|
|
||||||
RowIndex: int(e.Index),
|
|
||||||
}
|
|
||||||
for _, errp := range e.Errors {
|
|
||||||
rie.Errors = append(rie.Errors, bqToError(errp))
|
|
||||||
}
|
|
||||||
errs = append(errs, rie)
|
|
||||||
}
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
211
vendor/cloud.google.com/go/bigquery/uploader_test.go
generated
vendored
211
vendor/cloud.google.com/go/bigquery/uploader_test.go
generated
vendored
@ -1,211 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/pretty"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
type testSaver struct {
|
|
||||||
row map[string]Value
|
|
||||||
insertID string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ts testSaver) Save() (map[string]Value, string, error) {
|
|
||||||
return ts.row, ts.insertID, ts.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewInsertRequest(t *testing.T) {
|
|
||||||
prev := randomIDFn
|
|
||||||
n := 0
|
|
||||||
randomIDFn = func() string { n++; return strconv.Itoa(n) }
|
|
||||||
defer func() { randomIDFn = prev }()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
ul *Uploader
|
|
||||||
savers []ValueSaver
|
|
||||||
req *bq.TableDataInsertAllRequest
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
ul: &Uploader{},
|
|
||||||
savers: nil,
|
|
||||||
req: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ul: &Uploader{},
|
|
||||||
savers: []ValueSaver{
|
|
||||||
testSaver{row: map[string]Value{"one": 1}},
|
|
||||||
testSaver{row: map[string]Value{"two": 2}},
|
|
||||||
},
|
|
||||||
req: &bq.TableDataInsertAllRequest{
|
|
||||||
Rows: []*bq.TableDataInsertAllRequestRows{
|
|
||||||
{InsertId: "1", Json: map[string]bq.JsonValue{"one": 1}},
|
|
||||||
{InsertId: "2", Json: map[string]bq.JsonValue{"two": 2}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ul: &Uploader{
|
|
||||||
TableTemplateSuffix: "suffix",
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
SkipInvalidRows: true,
|
|
||||||
},
|
|
||||||
savers: []ValueSaver{
|
|
||||||
testSaver{insertID: "a", row: map[string]Value{"one": 1}},
|
|
||||||
testSaver{insertID: "", row: map[string]Value{"two": 2}},
|
|
||||||
},
|
|
||||||
req: &bq.TableDataInsertAllRequest{
|
|
||||||
Rows: []*bq.TableDataInsertAllRequestRows{
|
|
||||||
{InsertId: "a", Json: map[string]bq.JsonValue{"one": 1}},
|
|
||||||
{InsertId: "3", Json: map[string]bq.JsonValue{"two": 2}},
|
|
||||||
},
|
|
||||||
TemplateSuffix: "suffix",
|
|
||||||
SkipInvalidRows: true,
|
|
||||||
IgnoreUnknownValues: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for i, tc := range tests {
|
|
||||||
got, err := tc.ul.newInsertRequest(tc.savers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := tc.req
|
|
||||||
if !testutil.Equal(got, want) {
|
|
||||||
t.Errorf("%d: %#v: got %#v, want %#v", i, tc.ul, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewInsertRequestErrors(t *testing.T) {
|
|
||||||
var u Uploader
|
|
||||||
_, err := u.newInsertRequest([]ValueSaver{testSaver{err: errors.New("!")}})
|
|
||||||
if err == nil {
|
|
||||||
t.Error("got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleInsertErrors(t *testing.T) {
|
|
||||||
rows := []*bq.TableDataInsertAllRequestRows{
|
|
||||||
{InsertId: "a"},
|
|
||||||
{InsertId: "b"},
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
in []*bq.TableDataInsertAllResponseInsertErrors
|
|
||||||
want error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: nil,
|
|
||||||
want: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: []*bq.TableDataInsertAllResponseInsertErrors{{Index: 1}},
|
|
||||||
want: PutMultiError{RowInsertionError{InsertID: "b", RowIndex: 1}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: []*bq.TableDataInsertAllResponseInsertErrors{{Index: 1}},
|
|
||||||
want: PutMultiError{RowInsertionError{InsertID: "b", RowIndex: 1}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: []*bq.TableDataInsertAllResponseInsertErrors{
|
|
||||||
{Errors: []*bq.ErrorProto{{Message: "m0"}}, Index: 0},
|
|
||||||
{Errors: []*bq.ErrorProto{{Message: "m1"}}, Index: 1},
|
|
||||||
},
|
|
||||||
want: PutMultiError{
|
|
||||||
RowInsertionError{InsertID: "a", RowIndex: 0, Errors: []error{&Error{Message: "m0"}}},
|
|
||||||
RowInsertionError{InsertID: "b", RowIndex: 1, Errors: []error{&Error{Message: "m1"}}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
got := handleInsertErrors(test.in, rows)
|
|
||||||
if !testutil.Equal(got, test.want) {
|
|
||||||
t.Errorf("%#v:\ngot\n%#v\nwant\n%#v", test.in, got, test.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValueSavers(t *testing.T) {
|
|
||||||
ts := &testSaver{}
|
|
||||||
type T struct{ I int }
|
|
||||||
schema, err := InferSchema(T{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, test := range []struct {
|
|
||||||
in interface{}
|
|
||||||
want []ValueSaver
|
|
||||||
}{
|
|
||||||
{[]interface{}(nil), nil},
|
|
||||||
{[]interface{}{}, nil},
|
|
||||||
{ts, []ValueSaver{ts}},
|
|
||||||
{T{I: 1}, []ValueSaver{&StructSaver{Schema: schema, Struct: T{I: 1}}}},
|
|
||||||
{[]ValueSaver{ts, ts}, []ValueSaver{ts, ts}},
|
|
||||||
{[]interface{}{ts, ts}, []ValueSaver{ts, ts}},
|
|
||||||
{[]T{{I: 1}, {I: 2}}, []ValueSaver{
|
|
||||||
&StructSaver{Schema: schema, Struct: T{I: 1}},
|
|
||||||
&StructSaver{Schema: schema, Struct: T{I: 2}},
|
|
||||||
}},
|
|
||||||
{[]interface{}{T{I: 1}, &T{I: 2}}, []ValueSaver{
|
|
||||||
&StructSaver{Schema: schema, Struct: T{I: 1}},
|
|
||||||
&StructSaver{Schema: schema, Struct: &T{I: 2}},
|
|
||||||
}},
|
|
||||||
{&StructSaver{Struct: T{I: 3}, InsertID: "foo"},
|
|
||||||
[]ValueSaver{
|
|
||||||
&StructSaver{Schema: schema, Struct: T{I: 3}, InsertID: "foo"},
|
|
||||||
}},
|
|
||||||
} {
|
|
||||||
got, err := valueSavers(test.in)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !testutil.Equal(got, test.want, cmp.AllowUnexported(testSaver{})) {
|
|
||||||
t.Errorf("%+v: got %v, want %v", test.in, pretty.Value(got), pretty.Value(test.want))
|
|
||||||
}
|
|
||||||
// Make sure Save is successful.
|
|
||||||
for i, vs := range got {
|
|
||||||
_, _, err := vs.Save()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v, #%d: got error %v, want nil", test.in, i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValueSaversErrors(t *testing.T) {
|
|
||||||
inputs := []interface{}{
|
|
||||||
nil,
|
|
||||||
1,
|
|
||||||
[]int{1, 2},
|
|
||||||
[]interface{}{
|
|
||||||
testSaver{row: map[string]Value{"one": 1}, insertID: "a"},
|
|
||||||
1,
|
|
||||||
},
|
|
||||||
StructSaver{},
|
|
||||||
}
|
|
||||||
for _, in := range inputs {
|
|
||||||
if _, err := valueSavers(in); err == nil {
|
|
||||||
t.Errorf("%#v: got nil, want error", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
871
vendor/cloud.google.com/go/bigquery/value.go
generated
vendored
871
vendor/cloud.google.com/go/bigquery/value.go
generated
vendored
@ -1,871 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigquery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"math/big"
|
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/civil"
|
|
||||||
|
|
||||||
bq "google.golang.org/api/bigquery/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Value stores the contents of a single cell from a BigQuery result.
|
|
||||||
type Value interface{}
|
|
||||||
|
|
||||||
// ValueLoader stores a slice of Values representing a result row from a Read operation.
|
|
||||||
// See RowIterator.Next for more information.
|
|
||||||
type ValueLoader interface {
|
|
||||||
Load(v []Value, s Schema) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// valueList converts a []Value to implement ValueLoader.
|
|
||||||
type valueList []Value
|
|
||||||
|
|
||||||
// Load stores a sequence of values in a valueList.
|
|
||||||
// It resets the slice length to zero, then appends each value to it.
|
|
||||||
func (vs *valueList) Load(v []Value, _ Schema) error {
|
|
||||||
*vs = append((*vs)[:0], v...)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// valueMap converts a map[string]Value to implement ValueLoader.
|
|
||||||
type valueMap map[string]Value
|
|
||||||
|
|
||||||
// Load stores a sequence of values in a valueMap.
|
|
||||||
func (vm *valueMap) Load(v []Value, s Schema) error {
|
|
||||||
if *vm == nil {
|
|
||||||
*vm = map[string]Value{}
|
|
||||||
}
|
|
||||||
loadMap(*vm, v, s)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadMap(m map[string]Value, vals []Value, s Schema) {
|
|
||||||
for i, f := range s {
|
|
||||||
val := vals[i]
|
|
||||||
var v interface{}
|
|
||||||
switch {
|
|
||||||
case val == nil:
|
|
||||||
v = val
|
|
||||||
case f.Schema == nil:
|
|
||||||
v = val
|
|
||||||
case !f.Repeated:
|
|
||||||
m2 := map[string]Value{}
|
|
||||||
loadMap(m2, val.([]Value), f.Schema)
|
|
||||||
v = m2
|
|
||||||
default: // repeated and nested
|
|
||||||
sval := val.([]Value)
|
|
||||||
vs := make([]Value, len(sval))
|
|
||||||
for j, e := range sval {
|
|
||||||
m2 := map[string]Value{}
|
|
||||||
loadMap(m2, e.([]Value), f.Schema)
|
|
||||||
vs[j] = m2
|
|
||||||
}
|
|
||||||
v = vs
|
|
||||||
}
|
|
||||||
|
|
||||||
m[f.Name] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type structLoader struct {
|
|
||||||
typ reflect.Type // type of struct
|
|
||||||
err error
|
|
||||||
ops []structLoaderOp
|
|
||||||
|
|
||||||
vstructp reflect.Value // pointer to current struct value; changed by set
|
|
||||||
}
|
|
||||||
|
|
||||||
// A setFunc is a function that sets a struct field or slice/array
|
|
||||||
// element to a value.
|
|
||||||
type setFunc func(v reflect.Value, val interface{}) error
|
|
||||||
|
|
||||||
// A structLoaderOp instructs the loader to set a struct field to a row value.
|
|
||||||
type structLoaderOp struct {
|
|
||||||
fieldIndex []int
|
|
||||||
valueIndex int
|
|
||||||
setFunc setFunc
|
|
||||||
repeated bool
|
|
||||||
}
|
|
||||||
|
|
||||||
var errNoNulls = errors.New("bigquery: NULL values cannot be read into structs")
|
|
||||||
|
|
||||||
func setAny(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
v.Set(reflect.ValueOf(x))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setInt(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
xx := x.(int64)
|
|
||||||
if v.OverflowInt(xx) {
|
|
||||||
return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
|
|
||||||
}
|
|
||||||
v.SetInt(xx)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setUint(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
xx := x.(int64)
|
|
||||||
if xx < 0 || v.OverflowUint(uint64(xx)) {
|
|
||||||
return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
|
|
||||||
}
|
|
||||||
v.SetUint(uint64(xx))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setFloat(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
xx := x.(float64)
|
|
||||||
if v.OverflowFloat(xx) {
|
|
||||||
return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
|
|
||||||
}
|
|
||||||
v.SetFloat(xx)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setBool(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
v.SetBool(x.(bool))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setString(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
return errNoNulls
|
|
||||||
}
|
|
||||||
v.SetString(x.(string))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setBytes(v reflect.Value, x interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
v.SetBytes(nil)
|
|
||||||
} else {
|
|
||||||
v.SetBytes(x.([]byte))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setNull(v reflect.Value, x interface{}, build func() interface{}) error {
|
|
||||||
if x == nil {
|
|
||||||
v.Set(reflect.Zero(v.Type()))
|
|
||||||
} else {
|
|
||||||
n := build()
|
|
||||||
v.Set(reflect.ValueOf(n))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// set remembers a value for the next call to Load. The value must be
|
|
||||||
// a pointer to a struct. (This is checked in RowIterator.Next.)
|
|
||||||
func (sl *structLoader) set(structp interface{}, schema Schema) error {
|
|
||||||
if sl.err != nil {
|
|
||||||
return sl.err
|
|
||||||
}
|
|
||||||
sl.vstructp = reflect.ValueOf(structp)
|
|
||||||
typ := sl.vstructp.Type().Elem()
|
|
||||||
if sl.typ == nil {
|
|
||||||
// First call: remember the type and compile the schema.
|
|
||||||
sl.typ = typ
|
|
||||||
ops, err := compileToOps(typ, schema)
|
|
||||||
if err != nil {
|
|
||||||
sl.err = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sl.ops = ops
|
|
||||||
} else if sl.typ != typ {
|
|
||||||
return fmt.Errorf("bigquery: struct type changed from %s to %s", sl.typ, typ)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// compileToOps produces a sequence of operations that will set the fields of a
|
|
||||||
// value of structType to the contents of a row with schema.
|
|
||||||
func compileToOps(structType reflect.Type, schema Schema) ([]structLoaderOp, error) {
|
|
||||||
var ops []structLoaderOp
|
|
||||||
fields, err := fieldCache.Fields(structType)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for i, schemaField := range schema {
|
|
||||||
// Look for an exported struct field with the same name as the schema
|
|
||||||
// field, ignoring case (BigQuery column names are case-insensitive,
|
|
||||||
// and we want to act like encoding/json anyway).
|
|
||||||
structField := fields.Match(schemaField.Name)
|
|
||||||
if structField == nil {
|
|
||||||
// Ignore schema fields with no corresponding struct field.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
op := structLoaderOp{
|
|
||||||
fieldIndex: structField.Index,
|
|
||||||
valueIndex: i,
|
|
||||||
}
|
|
||||||
t := structField.Type
|
|
||||||
if schemaField.Repeated {
|
|
||||||
if t.Kind() != reflect.Slice && t.Kind() != reflect.Array {
|
|
||||||
return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but struct field %s has type %s",
|
|
||||||
schemaField.Name, structField.Name, t)
|
|
||||||
}
|
|
||||||
t = t.Elem()
|
|
||||||
op.repeated = true
|
|
||||||
}
|
|
||||||
if schemaField.Type == RecordFieldType {
|
|
||||||
// Field can be a struct or a pointer to a struct.
|
|
||||||
if t.Kind() == reflect.Ptr {
|
|
||||||
t = t.Elem()
|
|
||||||
}
|
|
||||||
if t.Kind() != reflect.Struct {
|
|
||||||
return nil, fmt.Errorf("bigquery: field %s has type %s, expected struct or *struct",
|
|
||||||
structField.Name, structField.Type)
|
|
||||||
}
|
|
||||||
nested, err := compileToOps(t, schemaField.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
op.setFunc = func(v reflect.Value, val interface{}) error {
|
|
||||||
return setNested(nested, v, val)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
op.setFunc = determineSetFunc(t, schemaField.Type)
|
|
||||||
if op.setFunc == nil {
|
|
||||||
return nil, fmt.Errorf("bigquery: schema field %s of type %s is not assignable to struct field %s of type %s",
|
|
||||||
schemaField.Name, schemaField.Type, structField.Name, t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ops = append(ops, op)
|
|
||||||
}
|
|
||||||
return ops, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// determineSetFunc chooses the best function for setting a field of type ftype
|
|
||||||
// to a value whose schema field type is stype. It returns nil if stype
|
|
||||||
// is not assignable to ftype.
|
|
||||||
// determineSetFunc considers only basic types. See compileToOps for
|
|
||||||
// handling of repetition and nesting.
|
|
||||||
func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
|
|
||||||
switch stype {
|
|
||||||
case StringFieldType:
|
|
||||||
if ftype.Kind() == reflect.String {
|
|
||||||
return setString
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullString {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullString{StringVal: x.(string), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case BytesFieldType:
|
|
||||||
if ftype == typeOfByteSlice {
|
|
||||||
return setBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
case IntegerFieldType:
|
|
||||||
if isSupportedUintType(ftype) {
|
|
||||||
return setUint
|
|
||||||
} else if isSupportedIntType(ftype) {
|
|
||||||
return setInt
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullInt64 {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullInt64{Int64: x.(int64), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case FloatFieldType:
|
|
||||||
switch ftype.Kind() {
|
|
||||||
case reflect.Float32, reflect.Float64:
|
|
||||||
return setFloat
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullFloat64 {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullFloat64{Float64: x.(float64), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case BooleanFieldType:
|
|
||||||
if ftype.Kind() == reflect.Bool {
|
|
||||||
return setBool
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullBool {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullBool{Bool: x.(bool), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case TimestampFieldType:
|
|
||||||
if ftype == typeOfGoTime {
|
|
||||||
return setAny
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullTimestamp {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullTimestamp{Timestamp: x.(time.Time), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case DateFieldType:
|
|
||||||
if ftype == typeOfDate {
|
|
||||||
return setAny
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullDate {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullDate{Date: x.(civil.Date), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case TimeFieldType:
|
|
||||||
if ftype == typeOfTime {
|
|
||||||
return setAny
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullTime {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullTime{Time: x.(civil.Time), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case DateTimeFieldType:
|
|
||||||
if ftype == typeOfDateTime {
|
|
||||||
return setAny
|
|
||||||
}
|
|
||||||
if ftype == typeOfNullDateTime {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} {
|
|
||||||
return NullDateTime{DateTime: x.(civil.DateTime), Valid: true}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case NumericFieldType:
|
|
||||||
if ftype == typeOfRat {
|
|
||||||
return func(v reflect.Value, x interface{}) error {
|
|
||||||
return setNull(v, x, func() interface{} { return x.(*big.Rat) })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sl *structLoader) Load(values []Value, _ Schema) error {
|
|
||||||
if sl.err != nil {
|
|
||||||
return sl.err
|
|
||||||
}
|
|
||||||
return runOps(sl.ops, sl.vstructp.Elem(), values)
|
|
||||||
}
|
|
||||||
|
|
||||||
// runOps executes a sequence of ops, setting the fields of vstruct to the
|
|
||||||
// supplied values.
|
|
||||||
func runOps(ops []structLoaderOp, vstruct reflect.Value, values []Value) error {
|
|
||||||
for _, op := range ops {
|
|
||||||
field := vstruct.FieldByIndex(op.fieldIndex)
|
|
||||||
var err error
|
|
||||||
if op.repeated {
|
|
||||||
err = setRepeated(field, values[op.valueIndex].([]Value), op.setFunc)
|
|
||||||
} else {
|
|
||||||
err = op.setFunc(field, values[op.valueIndex])
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func setNested(ops []structLoaderOp, v reflect.Value, val interface{}) error {
|
|
||||||
// v is either a struct or a pointer to a struct.
|
|
||||||
if v.Kind() == reflect.Ptr {
|
|
||||||
// If the value is nil, set the pointer to nil.
|
|
||||||
if val == nil {
|
|
||||||
v.Set(reflect.Zero(v.Type()))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// If the pointer is nil, set it to a zero struct value.
|
|
||||||
if v.IsNil() {
|
|
||||||
v.Set(reflect.New(v.Type().Elem()))
|
|
||||||
}
|
|
||||||
v = v.Elem()
|
|
||||||
}
|
|
||||||
return runOps(ops, v, val.([]Value))
|
|
||||||
}
|
|
||||||
|
|
||||||
func setRepeated(field reflect.Value, vslice []Value, setElem setFunc) error {
|
|
||||||
vlen := len(vslice)
|
|
||||||
var flen int
|
|
||||||
switch field.Type().Kind() {
|
|
||||||
case reflect.Slice:
|
|
||||||
// Make a slice of the right size, avoiding allocation if possible.
|
|
||||||
switch {
|
|
||||||
case field.Len() < vlen:
|
|
||||||
field.Set(reflect.MakeSlice(field.Type(), vlen, vlen))
|
|
||||||
case field.Len() > vlen:
|
|
||||||
field.SetLen(vlen)
|
|
||||||
}
|
|
||||||
flen = vlen
|
|
||||||
|
|
||||||
case reflect.Array:
|
|
||||||
flen = field.Len()
|
|
||||||
if flen > vlen {
|
|
||||||
// Set extra elements to their zero value.
|
|
||||||
z := reflect.Zero(field.Type().Elem())
|
|
||||||
for i := vlen; i < flen; i++ {
|
|
||||||
field.Index(i).Set(z)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("bigquery: impossible field type %s", field.Type())
|
|
||||||
}
|
|
||||||
for i, val := range vslice {
|
|
||||||
if i < flen { // avoid writing past the end of a short array
|
|
||||||
if err := setElem(field.Index(i), val); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// A ValueSaver returns a row of data to be inserted into a table.
|
|
||||||
type ValueSaver interface {
|
|
||||||
// Save returns a row to be inserted into a BigQuery table, represented
|
|
||||||
// as a map from field name to Value.
|
|
||||||
// If insertID is non-empty, BigQuery will use it to de-duplicate
|
|
||||||
// insertions of this row on a best-effort basis.
|
|
||||||
Save() (row map[string]Value, insertID string, err error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValuesSaver implements ValueSaver for a slice of Values.
|
|
||||||
type ValuesSaver struct {
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// If non-empty, BigQuery will use InsertID to de-duplicate insertions
|
|
||||||
// of this row on a best-effort basis.
|
|
||||||
InsertID string
|
|
||||||
|
|
||||||
Row []Value
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save implements ValueSaver.
|
|
||||||
func (vls *ValuesSaver) Save() (map[string]Value, string, error) {
|
|
||||||
m, err := valuesToMap(vls.Row, vls.Schema)
|
|
||||||
return m, vls.InsertID, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func valuesToMap(vs []Value, schema Schema) (map[string]Value, error) {
|
|
||||||
if len(vs) != len(schema) {
|
|
||||||
return nil, errors.New("Schema does not match length of row to be inserted")
|
|
||||||
}
|
|
||||||
|
|
||||||
m := make(map[string]Value)
|
|
||||||
for i, fieldSchema := range schema {
|
|
||||||
if vs[i] == nil {
|
|
||||||
m[fieldSchema.Name] = nil
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if fieldSchema.Type != RecordFieldType {
|
|
||||||
m[fieldSchema.Name] = toUploadValue(vs[i], fieldSchema)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Nested record, possibly repeated.
|
|
||||||
vals, ok := vs[i].([]Value)
|
|
||||||
if !ok {
|
|
||||||
return nil, errors.New("nested record is not a []Value")
|
|
||||||
}
|
|
||||||
if !fieldSchema.Repeated {
|
|
||||||
value, err := valuesToMap(vals, fieldSchema.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m[fieldSchema.Name] = value
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// A repeated nested field is converted into a slice of maps.
|
|
||||||
var maps []Value
|
|
||||||
for _, v := range vals {
|
|
||||||
sv, ok := v.([]Value)
|
|
||||||
if !ok {
|
|
||||||
return nil, errors.New("nested record in slice is not a []Value")
|
|
||||||
}
|
|
||||||
value, err := valuesToMap(sv, fieldSchema.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
maps = append(maps, value)
|
|
||||||
}
|
|
||||||
m[fieldSchema.Name] = maps
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// StructSaver implements ValueSaver for a struct.
|
|
||||||
// The struct is converted to a map of values by using the values of struct
|
|
||||||
// fields corresponding to schema fields. Additional and missing
|
|
||||||
// fields are ignored, as are nested struct pointers that are nil.
|
|
||||||
type StructSaver struct {
|
|
||||||
// Schema determines what fields of the struct are uploaded. It should
|
|
||||||
// match the table's schema.
|
|
||||||
// Schema is optional for StructSavers that are passed to Uploader.Put.
|
|
||||||
Schema Schema
|
|
||||||
|
|
||||||
// If non-empty, BigQuery will use InsertID to de-duplicate insertions
|
|
||||||
// of this row on a best-effort basis.
|
|
||||||
InsertID string
|
|
||||||
|
|
||||||
// Struct should be a struct or a pointer to a struct.
|
|
||||||
Struct interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save implements ValueSaver.
|
|
||||||
func (ss *StructSaver) Save() (row map[string]Value, insertID string, err error) {
|
|
||||||
vstruct := reflect.ValueOf(ss.Struct)
|
|
||||||
row, err = structToMap(vstruct, ss.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return row, ss.InsertID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func structToMap(vstruct reflect.Value, schema Schema) (map[string]Value, error) {
|
|
||||||
if vstruct.Kind() == reflect.Ptr {
|
|
||||||
vstruct = vstruct.Elem()
|
|
||||||
}
|
|
||||||
if !vstruct.IsValid() {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
m := map[string]Value{}
|
|
||||||
if vstruct.Kind() != reflect.Struct {
|
|
||||||
return nil, fmt.Errorf("bigquery: type is %s, need struct or struct pointer", vstruct.Type())
|
|
||||||
}
|
|
||||||
fields, err := fieldCache.Fields(vstruct.Type())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, schemaField := range schema {
|
|
||||||
// Look for an exported struct field with the same name as the schema
|
|
||||||
// field, ignoring case.
|
|
||||||
structField := fields.Match(schemaField.Name)
|
|
||||||
if structField == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
val, err := structFieldToUploadValue(vstruct.FieldByIndex(structField.Index), schemaField)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Add the value to the map, unless it is nil.
|
|
||||||
if val != nil {
|
|
||||||
m[schemaField.Name] = val
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// structFieldToUploadValue converts a struct field to a value suitable for ValueSaver.Save, using
|
|
||||||
// the schemaField as a guide.
|
|
||||||
// structFieldToUploadValue is careful to return a true nil interface{} when needed, so its
|
|
||||||
// caller can easily identify a nil value.
|
|
||||||
func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (interface{}, error) {
|
|
||||||
if schemaField.Repeated && (vfield.Kind() != reflect.Slice && vfield.Kind() != reflect.Array) {
|
|
||||||
return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but value has type %s",
|
|
||||||
schemaField.Name, vfield.Type())
|
|
||||||
}
|
|
||||||
|
|
||||||
// A non-nested field can be represented by its Go value, except for some types.
|
|
||||||
if schemaField.Type != RecordFieldType {
|
|
||||||
return toUploadValueReflect(vfield, schemaField), nil
|
|
||||||
}
|
|
||||||
// A non-repeated nested field is converted into a map[string]Value.
|
|
||||||
if !schemaField.Repeated {
|
|
||||||
m, err := structToMap(vfield, schemaField.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if m == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
// A repeated nested field is converted into a slice of maps.
|
|
||||||
if vfield.Len() == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var vals []Value
|
|
||||||
for i := 0; i < vfield.Len(); i++ {
|
|
||||||
m, err := structToMap(vfield.Index(i), schemaField.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
vals = append(vals, m)
|
|
||||||
}
|
|
||||||
return vals, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
|
|
||||||
if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
|
|
||||||
return toUploadValueReflect(reflect.ValueOf(val), fs)
|
|
||||||
}
|
|
||||||
return val
|
|
||||||
}
|
|
||||||
|
|
||||||
func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
|
|
||||||
switch fs.Type {
|
|
||||||
case TimeFieldType:
|
|
||||||
if v.Type() == typeOfNullTime {
|
|
||||||
return v.Interface()
|
|
||||||
}
|
|
||||||
return formatUploadValue(v, fs, func(v reflect.Value) string {
|
|
||||||
return CivilTimeString(v.Interface().(civil.Time))
|
|
||||||
})
|
|
||||||
case DateTimeFieldType:
|
|
||||||
if v.Type() == typeOfNullDateTime {
|
|
||||||
return v.Interface()
|
|
||||||
}
|
|
||||||
return formatUploadValue(v, fs, func(v reflect.Value) string {
|
|
||||||
return CivilDateTimeString(v.Interface().(civil.DateTime))
|
|
||||||
})
|
|
||||||
case NumericFieldType:
|
|
||||||
if r, ok := v.Interface().(*big.Rat); ok && r == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return formatUploadValue(v, fs, func(v reflect.Value) string {
|
|
||||||
return NumericString(v.Interface().(*big.Rat))
|
|
||||||
})
|
|
||||||
default:
|
|
||||||
if !fs.Repeated || v.Len() > 0 {
|
|
||||||
return v.Interface()
|
|
||||||
}
|
|
||||||
// The service treats a null repeated field as an error. Return
|
|
||||||
// nil to omit the field entirely.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatUploadValue(v reflect.Value, fs *FieldSchema, cvt func(reflect.Value) string) interface{} {
|
|
||||||
if !fs.Repeated {
|
|
||||||
return cvt(v)
|
|
||||||
}
|
|
||||||
if v.Len() == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
s := make([]string, v.Len())
|
|
||||||
for i := 0; i < v.Len(); i++ {
|
|
||||||
s[i] = cvt(v.Index(i))
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// CivilTimeString returns a string representing a civil.Time in a format compatible
|
|
||||||
// with BigQuery SQL. It rounds the time to the nearest microsecond and returns a
|
|
||||||
// string with six digits of sub-second precision.
|
|
||||||
//
|
|
||||||
// Use CivilTimeString when using civil.Time in DML, for example in INSERT
|
|
||||||
// statements.
|
|
||||||
func CivilTimeString(t civil.Time) string {
|
|
||||||
if t.Nanosecond == 0 {
|
|
||||||
return t.String()
|
|
||||||
} else {
|
|
||||||
micro := (t.Nanosecond + 500) / 1000 // round to nearest microsecond
|
|
||||||
t.Nanosecond = 0
|
|
||||||
return t.String() + fmt.Sprintf(".%06d", micro)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CivilDateTimeString returns a string representing a civil.DateTime in a format compatible
|
|
||||||
// with BigQuery SQL. It separate the date and time with a space, and formats the time
|
|
||||||
// with CivilTimeString.
|
|
||||||
//
|
|
||||||
// Use CivilDateTimeString when using civil.DateTime in DML, for example in INSERT
|
|
||||||
// statements.
|
|
||||||
func CivilDateTimeString(dt civil.DateTime) string {
|
|
||||||
return dt.Date.String() + " " + CivilTimeString(dt.Time)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCivilDateTime parses a date-time represented in a BigQuery SQL
|
|
||||||
// compatible format and returns a civil.DateTime.
|
|
||||||
func parseCivilDateTime(s string) (civil.DateTime, error) {
|
|
||||||
parts := strings.Fields(s)
|
|
||||||
if len(parts) != 2 {
|
|
||||||
return civil.DateTime{}, fmt.Errorf("bigquery: bad DATETIME value %q", s)
|
|
||||||
}
|
|
||||||
return civil.ParseDateTime(parts[0] + "T" + parts[1])
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
// The maximum number of digits in a NUMERIC value.
|
|
||||||
NumericPrecisionDigits = 38
|
|
||||||
|
|
||||||
// The maximum number of digits after the decimal point in a NUMERIC value.
|
|
||||||
NumericScaleDigits = 9
|
|
||||||
)
|
|
||||||
|
|
||||||
// NumericString returns a string representing a *big.Rat in a format compatible
|
|
||||||
// with BigQuery SQL. It returns a floating-point literal with 9 digits
|
|
||||||
// after the decimal point.
|
|
||||||
func NumericString(r *big.Rat) string {
|
|
||||||
return r.FloatString(NumericScaleDigits)
|
|
||||||
}
|
|
||||||
|
|
||||||
// convertRows converts a series of TableRows into a series of Value slices.
|
|
||||||
// schema is used to interpret the data from rows; its length must match the
|
|
||||||
// length of each row.
|
|
||||||
func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) {
|
|
||||||
var rs [][]Value
|
|
||||||
for _, r := range rows {
|
|
||||||
row, err := convertRow(r, schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rs = append(rs, row)
|
|
||||||
}
|
|
||||||
return rs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) {
|
|
||||||
if len(schema) != len(r.F) {
|
|
||||||
return nil, errors.New("schema length does not match row length")
|
|
||||||
}
|
|
||||||
var values []Value
|
|
||||||
for i, cell := range r.F {
|
|
||||||
fs := schema[i]
|
|
||||||
v, err := convertValue(cell.V, fs.Type, fs.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
values = append(values, v)
|
|
||||||
}
|
|
||||||
return values, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) {
|
|
||||||
switch val := val.(type) {
|
|
||||||
case nil:
|
|
||||||
return nil, nil
|
|
||||||
case []interface{}:
|
|
||||||
return convertRepeatedRecord(val, typ, schema)
|
|
||||||
case map[string]interface{}:
|
|
||||||
return convertNestedRecord(val, schema)
|
|
||||||
case string:
|
|
||||||
return convertBasicType(val, typ)
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) {
|
|
||||||
var values []Value
|
|
||||||
for _, cell := range vals {
|
|
||||||
// each cell contains a single entry, keyed by "v"
|
|
||||||
val := cell.(map[string]interface{})["v"]
|
|
||||||
v, err := convertValue(val, typ, schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
values = append(values, v)
|
|
||||||
}
|
|
||||||
return values, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) {
|
|
||||||
// convertNestedRecord is similar to convertRow, as a record has the same structure as a row.
|
|
||||||
|
|
||||||
// Nested records are wrapped in a map with a single key, "f".
|
|
||||||
record := val["f"].([]interface{})
|
|
||||||
if len(record) != len(schema) {
|
|
||||||
return nil, errors.New("schema length does not match record length")
|
|
||||||
}
|
|
||||||
|
|
||||||
var values []Value
|
|
||||||
for i, cell := range record {
|
|
||||||
// each cell contains a single entry, keyed by "v"
|
|
||||||
val := cell.(map[string]interface{})["v"]
|
|
||||||
fs := schema[i]
|
|
||||||
v, err := convertValue(val, fs.Type, fs.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
values = append(values, v)
|
|
||||||
}
|
|
||||||
return values, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// convertBasicType returns val as an interface with a concrete type specified by typ.
|
|
||||||
func convertBasicType(val string, typ FieldType) (Value, error) {
|
|
||||||
switch typ {
|
|
||||||
case StringFieldType:
|
|
||||||
return val, nil
|
|
||||||
case BytesFieldType:
|
|
||||||
return base64.StdEncoding.DecodeString(val)
|
|
||||||
case IntegerFieldType:
|
|
||||||
return strconv.ParseInt(val, 10, 64)
|
|
||||||
case FloatFieldType:
|
|
||||||
return strconv.ParseFloat(val, 64)
|
|
||||||
case BooleanFieldType:
|
|
||||||
return strconv.ParseBool(val)
|
|
||||||
case TimestampFieldType:
|
|
||||||
f, err := strconv.ParseFloat(val, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
secs := math.Trunc(f)
|
|
||||||
nanos := (f - secs) * 1e9
|
|
||||||
return Value(time.Unix(int64(secs), int64(nanos)).UTC()), nil
|
|
||||||
case DateFieldType:
|
|
||||||
return civil.ParseDate(val)
|
|
||||||
case TimeFieldType:
|
|
||||||
return civil.ParseTime(val)
|
|
||||||
case DateTimeFieldType:
|
|
||||||
return civil.ParseDateTime(val)
|
|
||||||
case NumericFieldType:
|
|
||||||
r, ok := (&big.Rat{}).SetString(val)
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
|
|
||||||
}
|
|
||||||
return Value(r), nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unrecognized type: %s", typ)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
1210
vendor/cloud.google.com/go/bigquery/value_test.go
generated
vendored
1210
vendor/cloud.google.com/go/bigquery/value_test.go
generated
vendored
File diff suppressed because it is too large
Load Diff
1092
vendor/cloud.google.com/go/bigtable/admin.go
generated
vendored
1092
vendor/cloud.google.com/go/bigtable/admin.go
generated
vendored
File diff suppressed because it is too large
Load Diff
578
vendor/cloud.google.com/go/bigtable/admin_test.go
generated
vendored
578
vendor/cloud.google.com/go/bigtable/admin_test.go
generated
vendored
@ -1,578 +0,0 @@
|
|||||||
// Copyright 2015 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bigtable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"github.com/golang/protobuf/proto"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/iterator"
|
|
||||||
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAdminIntegration(t *testing.T) {
|
|
||||||
testEnv, err := NewIntegrationEnv()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IntegrationEnv: %v", err)
|
|
||||||
}
|
|
||||||
defer testEnv.Close()
|
|
||||||
|
|
||||||
timeout := 2 * time.Second
|
|
||||||
if testEnv.Config().UseProd {
|
|
||||||
timeout = 5 * time.Minute
|
|
||||||
}
|
|
||||||
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
||||||
|
|
||||||
adminClient, err := testEnv.NewAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
defer adminClient.Close()
|
|
||||||
|
|
||||||
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
if iAdminClient != nil {
|
|
||||||
defer iAdminClient.Close()
|
|
||||||
|
|
||||||
iInfo, err := iAdminClient.InstanceInfo(ctx, adminClient.instance)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("InstanceInfo: %v", err)
|
|
||||||
}
|
|
||||||
if iInfo.Name != adminClient.instance {
|
|
||||||
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
list := func() []string {
|
|
||||||
tbls, err := adminClient.Tables(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Fetching list of tables: %v", err)
|
|
||||||
}
|
|
||||||
sort.Strings(tbls)
|
|
||||||
return tbls
|
|
||||||
}
|
|
||||||
containsAll := func(got, want []string) bool {
|
|
||||||
gotSet := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, s := range got {
|
|
||||||
gotSet[s] = true
|
|
||||||
}
|
|
||||||
for _, s := range want {
|
|
||||||
if !gotSet[s] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
defer adminClient.DeleteTable(ctx, "mytable")
|
|
||||||
|
|
||||||
if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer adminClient.DeleteTable(ctx, "myothertable")
|
|
||||||
|
|
||||||
if err := adminClient.CreateTable(ctx, "myothertable"); err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got, want := list(), []string{"myothertable", "mytable"}; !containsAll(got, want) {
|
|
||||||
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
must(adminClient.WaitForReplication(ctx, "mytable"))
|
|
||||||
|
|
||||||
if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil {
|
|
||||||
t.Fatalf("Deleting table: %v", err)
|
|
||||||
}
|
|
||||||
tables := list()
|
|
||||||
if got, want := tables, []string{"mytable"}; !containsAll(got, want) {
|
|
||||||
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if got, unwanted := tables, []string{"myothertable"}; containsAll(got, unwanted) {
|
|
||||||
t.Errorf("adminClient.Tables return %#v. unwanted %#v", got, unwanted)
|
|
||||||
}
|
|
||||||
|
|
||||||
tblConf := TableConf{
|
|
||||||
TableID: "conftable",
|
|
||||||
Families: map[string]GCPolicy{
|
|
||||||
"fam1": MaxVersionsPolicy(1),
|
|
||||||
"fam2": MaxVersionsPolicy(2),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := adminClient.CreateTableFromConf(ctx, &tblConf); err != nil {
|
|
||||||
t.Fatalf("Creating table from TableConf: %v", err)
|
|
||||||
}
|
|
||||||
defer adminClient.DeleteTable(ctx, tblConf.TableID)
|
|
||||||
|
|
||||||
tblInfo, err := adminClient.TableInfo(ctx, tblConf.TableID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Getting table info: %v", err)
|
|
||||||
}
|
|
||||||
sort.Strings(tblInfo.Families)
|
|
||||||
wantFams := []string{"fam1", "fam2"}
|
|
||||||
if !testutil.Equal(tblInfo.Families, wantFams) {
|
|
||||||
t.Errorf("Column family mismatch, got %v, want %v", tblInfo.Families, wantFams)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Populate mytable and drop row ranges
|
|
||||||
if err = adminClient.CreateColumnFamily(ctx, "mytable", "cf"); err != nil {
|
|
||||||
t.Fatalf("Creating column family: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := testEnv.NewClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient: %v", err)
|
|
||||||
}
|
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
tbl := client.Open("mytable")
|
|
||||||
|
|
||||||
prefixes := []string{"a", "b", "c"}
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
for i := 0; i < 5; i++ {
|
|
||||||
mut := NewMutation()
|
|
||||||
mut.Set("cf", "col", 1000, []byte("1"))
|
|
||||||
if err := tbl.Apply(ctx, fmt.Sprintf("%v-%v", prefix, i), mut); err != nil {
|
|
||||||
t.Fatalf("Mutating row: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = adminClient.DropRowRange(ctx, "mytable", "a"); err != nil {
|
|
||||||
t.Errorf("DropRowRange a: %v", err)
|
|
||||||
}
|
|
||||||
if err = adminClient.DropRowRange(ctx, "mytable", "c"); err != nil {
|
|
||||||
t.Errorf("DropRowRange c: %v", err)
|
|
||||||
}
|
|
||||||
if err = adminClient.DropRowRange(ctx, "mytable", "x"); err != nil {
|
|
||||||
t.Errorf("DropRowRange x: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var gotRowCount int
|
|
||||||
must(tbl.ReadRows(ctx, RowRange{}, func(row Row) bool {
|
|
||||||
gotRowCount += 1
|
|
||||||
if !strings.HasPrefix(row.Key(), "b") {
|
|
||||||
t.Errorf("Invalid row after dropping range: %v", row)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}))
|
|
||||||
if gotRowCount != 5 {
|
|
||||||
t.Errorf("Invalid row count after dropping range: got %v, want %v", gotRowCount, 5)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInstanceUpdate(t *testing.T) {
|
|
||||||
testEnv, err := NewIntegrationEnv()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IntegrationEnv: %v", err)
|
|
||||||
}
|
|
||||||
defer testEnv.Close()
|
|
||||||
|
|
||||||
timeout := 2 * time.Second
|
|
||||||
if testEnv.Config().UseProd {
|
|
||||||
timeout = 5 * time.Minute
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
adminClient, err := testEnv.NewAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer adminClient.Close()
|
|
||||||
|
|
||||||
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iAdminClient == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
defer iAdminClient.Close()
|
|
||||||
|
|
||||||
iInfo, err := iAdminClient.InstanceInfo(ctx, adminClient.instance)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("InstanceInfo: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iInfo.Name != adminClient.instance {
|
|
||||||
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iInfo.DisplayName != adminClient.instance {
|
|
||||||
t.Errorf("InstanceInfo returned name %#v, want %#v", iInfo.Name, adminClient.instance)
|
|
||||||
}
|
|
||||||
|
|
||||||
const numNodes = 4
|
|
||||||
// update cluster nodes
|
|
||||||
if err := iAdminClient.UpdateCluster(ctx, adminClient.instance, testEnv.Config().Cluster, int32(numNodes)); err != nil {
|
|
||||||
t.Errorf("UpdateCluster: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// get cluster after updating
|
|
||||||
cis, err := iAdminClient.GetCluster(ctx, adminClient.instance, testEnv.Config().Cluster)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("GetCluster %v", err)
|
|
||||||
}
|
|
||||||
if cis.ServeNodes != int(numNodes) {
|
|
||||||
t.Errorf("ServeNodes returned %d, want %d", cis.ServeNodes, int(numNodes))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdminSnapshotIntegration(t *testing.T) {
|
|
||||||
testEnv, err := NewIntegrationEnv()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IntegrationEnv: %v", err)
|
|
||||||
}
|
|
||||||
defer testEnv.Close()
|
|
||||||
|
|
||||||
if !testEnv.Config().UseProd {
|
|
||||||
t.Skip("emulator doesn't support snapshots")
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := 2 * time.Second
|
|
||||||
if testEnv.Config().UseProd {
|
|
||||||
timeout = 5 * time.Minute
|
|
||||||
}
|
|
||||||
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
||||||
|
|
||||||
adminClient, err := testEnv.NewAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
defer adminClient.Close()
|
|
||||||
|
|
||||||
table := testEnv.Config().Table
|
|
||||||
cluster := testEnv.Config().Cluster
|
|
||||||
|
|
||||||
list := func(cluster string) ([]*SnapshotInfo, error) {
|
|
||||||
infos := []*SnapshotInfo(nil)
|
|
||||||
|
|
||||||
it := adminClient.Snapshots(ctx, cluster)
|
|
||||||
for {
|
|
||||||
s, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
infos = append(infos, s)
|
|
||||||
}
|
|
||||||
return infos, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete the table at the end of the test. Schedule ahead of time
|
|
||||||
// in case the client fails
|
|
||||||
defer adminClient.DeleteTable(ctx, table)
|
|
||||||
|
|
||||||
if err := adminClient.CreateTable(ctx, table); err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Precondition: no snapshots
|
|
||||||
snapshots, err := list(cluster)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Initial snapshot list: %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(snapshots), 0; got != want {
|
|
||||||
t.Fatalf("Initial snapshot list len: %d, want: %d", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create snapshot
|
|
||||||
defer adminClient.DeleteSnapshot(ctx, cluster, "mysnapshot")
|
|
||||||
|
|
||||||
if err = adminClient.SnapshotTable(ctx, table, cluster, "mysnapshot", 5*time.Hour); err != nil {
|
|
||||||
t.Fatalf("Creating snaphot: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List snapshot
|
|
||||||
snapshots, err = list(cluster)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Listing snapshots: %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(snapshots), 1; got != want {
|
|
||||||
t.Fatalf("Listing snapshot count: %d, want: %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := snapshots[0].Name, "mysnapshot"; got != want {
|
|
||||||
t.Fatalf("Snapshot name: %s, want: %s", got, want)
|
|
||||||
}
|
|
||||||
if got, want := snapshots[0].SourceTable, table; got != want {
|
|
||||||
t.Fatalf("Snapshot SourceTable: %s, want: %s", got, want)
|
|
||||||
}
|
|
||||||
if got, want := snapshots[0].DeleteTime, snapshots[0].CreateTime.Add(5*time.Hour); math.Abs(got.Sub(want).Minutes()) > 1 {
|
|
||||||
t.Fatalf("Snapshot DeleteTime: %s, want: %s", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get snapshot
|
|
||||||
snapshot, err := adminClient.SnapshotInfo(ctx, cluster, "mysnapshot")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("SnapshotInfo: %v", snapshot)
|
|
||||||
}
|
|
||||||
if got, want := *snapshot, *snapshots[0]; got != want {
|
|
||||||
t.Fatalf("SnapshotInfo: %v, want: %v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Restore
|
|
||||||
restoredTable := table + "-restored"
|
|
||||||
defer adminClient.DeleteTable(ctx, restoredTable)
|
|
||||||
if err = adminClient.CreateTableFromSnapshot(ctx, restoredTable, cluster, "mysnapshot"); err != nil {
|
|
||||||
t.Fatalf("CreateTableFromSnapshot: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := adminClient.TableInfo(ctx, restoredTable); err != nil {
|
|
||||||
t.Fatalf("Restored TableInfo: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete snapshot
|
|
||||||
if err = adminClient.DeleteSnapshot(ctx, cluster, "mysnapshot"); err != nil {
|
|
||||||
t.Fatalf("DeleteSnapshot: %v", err)
|
|
||||||
}
|
|
||||||
snapshots, err = list(cluster)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("List after Delete: %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(snapshots), 0; got != want {
|
|
||||||
t.Fatalf("List after delete len: %d, want: %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGranularity(t *testing.T) {
|
|
||||||
testEnv, err := NewIntegrationEnv()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IntegrationEnv: %v", err)
|
|
||||||
}
|
|
||||||
defer testEnv.Close()
|
|
||||||
|
|
||||||
timeout := 2 * time.Second
|
|
||||||
if testEnv.Config().UseProd {
|
|
||||||
timeout = 5 * time.Minute
|
|
||||||
}
|
|
||||||
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
|
||||||
|
|
||||||
adminClient, err := testEnv.NewAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
defer adminClient.Close()
|
|
||||||
|
|
||||||
list := func() []string {
|
|
||||||
tbls, err := adminClient.Tables(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Fetching list of tables: %v", err)
|
|
||||||
}
|
|
||||||
sort.Strings(tbls)
|
|
||||||
return tbls
|
|
||||||
}
|
|
||||||
containsAll := func(got, want []string) bool {
|
|
||||||
gotSet := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, s := range got {
|
|
||||||
gotSet[s] = true
|
|
||||||
}
|
|
||||||
for _, s := range want {
|
|
||||||
if !gotSet[s] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
defer adminClient.DeleteTable(ctx, "mytable")
|
|
||||||
|
|
||||||
if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tables := list()
|
|
||||||
if got, want := tables, []string{"mytable"}; !containsAll(got, want) {
|
|
||||||
t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// calling ModifyColumnFamilies to check the granularity of table
|
|
||||||
prefix := adminClient.instancePrefix()
|
|
||||||
req := &btapb.ModifyColumnFamiliesRequest{
|
|
||||||
Name: prefix + "/tables/" + "mytable",
|
|
||||||
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
||||||
Id: "cf",
|
|
||||||
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
table, err := adminClient.tClient.ModifyColumnFamilies(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating column family: %v", err)
|
|
||||||
}
|
|
||||||
if table.Granularity != btapb.Table_TimestampGranularity(btapb.Table_MILLIS) {
|
|
||||||
t.Errorf("ModifyColumnFamilies returned granularity %#v, want %#v", table.Granularity, btapb.Table_TimestampGranularity(btapb.Table_MILLIS))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInstanceAdminClient_AppProfile(t *testing.T) {
|
|
||||||
testEnv, err := NewIntegrationEnv()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("IntegrationEnv: %v", err)
|
|
||||||
}
|
|
||||||
defer testEnv.Close()
|
|
||||||
|
|
||||||
timeout := 2 * time.Second
|
|
||||||
if testEnv.Config().UseProd {
|
|
||||||
timeout = 5 * time.Minute
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
adminClient, err := testEnv.NewAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
defer adminClient.Close()
|
|
||||||
|
|
||||||
iAdminClient, err := testEnv.NewInstanceAdminClient()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewInstanceAdminClient: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iAdminClient == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
defer iAdminClient.Close()
|
|
||||||
profile := ProfileConf{
|
|
||||||
ProfileID: "app_profile1",
|
|
||||||
InstanceID: adminClient.instance,
|
|
||||||
ClusterID: testEnv.Config().Cluster,
|
|
||||||
Description: "creating new app profile 1",
|
|
||||||
RoutingPolicy: SingleClusterRouting,
|
|
||||||
}
|
|
||||||
|
|
||||||
createdProfile, err := iAdminClient.CreateAppProfile(ctx, profile)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating app profile: %v", err)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
gotProfile, err := iAdminClient.GetAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get app profile: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !proto.Equal(createdProfile, gotProfile) {
|
|
||||||
t.Fatalf("created profile: %s, got profile: %s", createdProfile.Name, gotProfile.Name)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
list := func(instanceID string) ([]*btapb.AppProfile, error) {
|
|
||||||
profiles := []*btapb.AppProfile(nil)
|
|
||||||
|
|
||||||
it := iAdminClient.ListAppProfiles(ctx, instanceID)
|
|
||||||
for {
|
|
||||||
s, err := it.Next()
|
|
||||||
if err == iterator.Done {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
profiles = append(profiles, s)
|
|
||||||
}
|
|
||||||
return profiles, err
|
|
||||||
}
|
|
||||||
|
|
||||||
profiles, err := list(adminClient.instance)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("List app profile: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got, want := len(profiles), 1; got != want {
|
|
||||||
t.Fatalf("Initial app profile list len: %d, want: %d", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range []struct {
|
|
||||||
desc string
|
|
||||||
uattrs ProfileAttrsToUpdate
|
|
||||||
want *btapb.AppProfile // nil means error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
desc: "empty update",
|
|
||||||
uattrs: ProfileAttrsToUpdate{},
|
|
||||||
want: nil,
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
desc: "empty description update",
|
|
||||||
uattrs: ProfileAttrsToUpdate{Description: ""},
|
|
||||||
want: &btapb.AppProfile{
|
|
||||||
Name: gotProfile.Name,
|
|
||||||
Description: "",
|
|
||||||
RoutingPolicy: gotProfile.RoutingPolicy,
|
|
||||||
Etag: gotProfile.Etag},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
desc: "routing update",
|
|
||||||
uattrs: ProfileAttrsToUpdate{
|
|
||||||
RoutingPolicy: SingleClusterRouting,
|
|
||||||
ClusterID: testEnv.Config().Cluster,
|
|
||||||
},
|
|
||||||
want: &btapb.AppProfile{
|
|
||||||
Name: gotProfile.Name,
|
|
||||||
Description: "",
|
|
||||||
Etag: gotProfile.Etag,
|
|
||||||
RoutingPolicy: &btapb.AppProfile_SingleClusterRouting_{
|
|
||||||
SingleClusterRouting: &btapb.AppProfile_SingleClusterRouting{
|
|
||||||
ClusterId: testEnv.Config().Cluster,
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
err = iAdminClient.UpdateAppProfile(ctx, adminClient.instance, "app_profile1", test.uattrs)
|
|
||||||
if err != nil {
|
|
||||||
if test.want != nil {
|
|
||||||
t.Errorf("%s: %v", test.desc, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err == nil && test.want == nil {
|
|
||||||
t.Errorf("%s: got nil, want error", test.desc)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
got, _ := iAdminClient.GetAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
||||||
|
|
||||||
if !proto.Equal(got, test.want) {
|
|
||||||
t.Fatalf("%s : got profile : %v, want profile: %v", test.desc, gotProfile, test.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
err = iAdminClient.DeleteAppProfile(ctx, adminClient.instance, "app_profile1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Delete app profile: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
887
vendor/cloud.google.com/go/bigtable/bigtable.go
generated
vendored
887
vendor/cloud.google.com/go/bigtable/bigtable.go
generated
vendored
@ -1,887 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2015 Google LLC
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package bigtable // import "cloud.google.com/go/bigtable"
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigtable/internal/gax"
|
|
||||||
btopt "cloud.google.com/go/bigtable/internal/option"
|
|
||||||
"github.com/golang/protobuf/proto"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
gtransport "google.golang.org/api/transport/grpc"
|
|
||||||
btpb "google.golang.org/genproto/googleapis/bigtable/v2"
|
|
||||||
"google.golang.org/grpc"
|
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
"google.golang.org/grpc/metadata"
|
|
||||||
"google.golang.org/grpc/status"
|
|
||||||
)
|
|
||||||
|
|
||||||
const prodAddr = "bigtable.googleapis.com:443"
|
|
||||||
|
|
||||||
// Client is a client for reading and writing data to tables in an instance.
|
|
||||||
//
|
|
||||||
// A Client is safe to use concurrently, except for its Close method.
|
|
||||||
type Client struct {
|
|
||||||
conn *grpc.ClientConn
|
|
||||||
client btpb.BigtableClient
|
|
||||||
project, instance string
|
|
||||||
// App Profiles are part of the private alpha release of Cloud Bigtable replication.
|
|
||||||
// This feature
|
|
||||||
// is not currently available to most Cloud Bigtable customers. This feature
|
|
||||||
// might be changed in backward-incompatible ways and is not recommended for
|
|
||||||
// production use. It is not subject to any SLA or deprecation policy.
|
|
||||||
appProfile string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ClientConfig has configurations for the client.
|
|
||||||
type ClientConfig struct {
|
|
||||||
// The id of the app profile to associate with all data operations sent from this client.
|
|
||||||
// If unspecified, the default app profile for the instance will be used.
|
|
||||||
AppProfile string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient creates a new Client for a given project and instance.
|
|
||||||
// The default ClientConfig will be used.
|
|
||||||
func NewClient(ctx context.Context, project, instance string, opts ...option.ClientOption) (*Client, error) {
|
|
||||||
return NewClientWithConfig(ctx, project, instance, ClientConfig{}, opts...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewClientWithConfig(ctx context.Context, project, instance string, config ClientConfig, opts ...option.ClientOption) (*Client, error) {
|
|
||||||
o, err := btopt.DefaultClientOptions(prodAddr, Scope, clientUserAgent)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Default to a small connection pool that can be overridden.
|
|
||||||
o = append(o,
|
|
||||||
option.WithGRPCConnectionPool(4),
|
|
||||||
// Set the max size to correspond to server-side limits.
|
|
||||||
option.WithGRPCDialOption(grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(100<<20), grpc.MaxCallRecvMsgSize(100<<20))),
|
|
||||||
// TODO(grpc/grpc-go#1388) using connection pool without WithBlock
|
|
||||||
// can cause RPCs to fail randomly. We can delete this after the issue is fixed.
|
|
||||||
option.WithGRPCDialOption(grpc.WithBlock()))
|
|
||||||
o = append(o, opts...)
|
|
||||||
conn, err := gtransport.Dial(ctx, o...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("dialing: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Client{
|
|
||||||
conn: conn,
|
|
||||||
client: btpb.NewBigtableClient(conn),
|
|
||||||
project: project,
|
|
||||||
instance: instance,
|
|
||||||
appProfile: config.AppProfile,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the Client.
|
|
||||||
func (c *Client) Close() error {
|
|
||||||
return c.conn.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
idempotentRetryCodes = []codes.Code{codes.DeadlineExceeded, codes.Unavailable, codes.Aborted}
|
|
||||||
isIdempotentRetryCode = make(map[codes.Code]bool)
|
|
||||||
retryOptions = []gax.CallOption{
|
|
||||||
gax.WithDelayTimeoutSettings(100*time.Millisecond, 2000*time.Millisecond, 1.2),
|
|
||||||
gax.WithRetryCodes(idempotentRetryCodes),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
for _, code := range idempotentRetryCodes {
|
|
||||||
isIdempotentRetryCode[code] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Client) fullTableName(table string) string {
|
|
||||||
return fmt.Sprintf("projects/%s/instances/%s/tables/%s", c.project, c.instance, table)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A Table refers to a table.
|
|
||||||
//
|
|
||||||
// A Table is safe to use concurrently.
|
|
||||||
type Table struct {
|
|
||||||
c *Client
|
|
||||||
table string
|
|
||||||
|
|
||||||
// Metadata to be sent with each request.
|
|
||||||
md metadata.MD
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open opens a table.
|
|
||||||
func (c *Client) Open(table string) *Table {
|
|
||||||
return &Table{
|
|
||||||
c: c,
|
|
||||||
table: table,
|
|
||||||
md: metadata.Pairs(resourcePrefixHeader, c.fullTableName(table)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO(dsymonds): Read method that returns a sequence of ReadItems.
|
|
||||||
|
|
||||||
// ReadRows reads rows from a table. f is called for each row.
|
|
||||||
// If f returns false, the stream is shut down and ReadRows returns.
|
|
||||||
// f owns its argument, and f is called serially in order by row key.
|
|
||||||
//
|
|
||||||
// By default, the yielded rows will contain all values in all cells.
|
|
||||||
// Use RowFilter to limit the cells returned.
|
|
||||||
func (t *Table) ReadRows(ctx context.Context, arg RowSet, f func(Row) bool, opts ...ReadOption) error {
|
|
||||||
ctx = mergeOutgoingMetadata(ctx, t.md)
|
|
||||||
|
|
||||||
var prevRowKey string
|
|
||||||
var err error
|
|
||||||
ctx = traceStartSpan(ctx, "cloud.google.com/go/bigtable.ReadRows")
|
|
||||||
defer func() { traceEndSpan(ctx, err) }()
|
|
||||||
attrMap := make(map[string]interface{})
|
|
||||||
err = gax.Invoke(ctx, func(ctx context.Context) error {
|
|
||||||
if !arg.valid() {
|
|
||||||
// Empty row set, no need to make an API call.
|
|
||||||
// NOTE: we must return early if arg == RowList{} because reading
|
|
||||||
// an empty RowList from bigtable returns all rows from that table.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
req := &btpb.ReadRowsRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
Rows: arg.proto(),
|
|
||||||
}
|
|
||||||
for _, opt := range opts {
|
|
||||||
opt.set(req)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(ctx) // for aborting the stream
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
startTime := time.Now()
|
|
||||||
stream, err := t.c.client.ReadRows(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
cr := newChunkReader()
|
|
||||||
for {
|
|
||||||
res, err := stream.Recv()
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
// Reset arg for next Invoke call.
|
|
||||||
arg = arg.retainRowsAfter(prevRowKey)
|
|
||||||
attrMap["rowKey"] = prevRowKey
|
|
||||||
attrMap["error"] = err.Error()
|
|
||||||
attrMap["time_secs"] = time.Since(startTime).Seconds()
|
|
||||||
tracePrintf(ctx, attrMap, "Retry details in ReadRows")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
attrMap["time_secs"] = time.Since(startTime).Seconds()
|
|
||||||
attrMap["rowCount"] = len(res.Chunks)
|
|
||||||
tracePrintf(ctx, attrMap, "Details in ReadRows")
|
|
||||||
|
|
||||||
for _, cc := range res.Chunks {
|
|
||||||
row, err := cr.Process(cc)
|
|
||||||
if err != nil {
|
|
||||||
// No need to prepare for a retry, this is an unretryable error.
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if row == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
prevRowKey = row.Key()
|
|
||||||
if !f(row) {
|
|
||||||
// Cancel and drain stream.
|
|
||||||
cancel()
|
|
||||||
for {
|
|
||||||
if _, err := stream.Recv(); err != nil {
|
|
||||||
// The stream has ended. We don't return an error
|
|
||||||
// because the caller has intentionally interrupted the scan.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := cr.Close(); err != nil {
|
|
||||||
// No need to prepare for a retry, this is an unretryable error.
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}, retryOptions...)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadRow is a convenience implementation of a single-row reader.
|
|
||||||
// A missing row will return a zero-length map and a nil error.
|
|
||||||
func (t *Table) ReadRow(ctx context.Context, row string, opts ...ReadOption) (Row, error) {
|
|
||||||
var r Row
|
|
||||||
err := t.ReadRows(ctx, SingleRow(row), func(rr Row) bool {
|
|
||||||
r = rr
|
|
||||||
return true
|
|
||||||
}, opts...)
|
|
||||||
return r, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeFamilyProto adds the cell data from f to the given row.
|
|
||||||
func decodeFamilyProto(r Row, row string, f *btpb.Family) {
|
|
||||||
fam := f.Name // does not have colon
|
|
||||||
for _, col := range f.Columns {
|
|
||||||
for _, cell := range col.Cells {
|
|
||||||
ri := ReadItem{
|
|
||||||
Row: row,
|
|
||||||
Column: fam + ":" + string(col.Qualifier),
|
|
||||||
Timestamp: Timestamp(cell.TimestampMicros),
|
|
||||||
Value: cell.Value,
|
|
||||||
}
|
|
||||||
r[fam] = append(r[fam], ri)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RowSet is a set of rows to be read. It is satisfied by RowList, RowRange and RowRangeList.
|
|
||||||
// The serialized size of the RowSet must be no larger than 1MiB.
|
|
||||||
type RowSet interface {
|
|
||||||
proto() *btpb.RowSet
|
|
||||||
|
|
||||||
// retainRowsAfter returns a new RowSet that does not include the
|
|
||||||
// given row key or any row key lexicographically less than it.
|
|
||||||
retainRowsAfter(lastRowKey string) RowSet
|
|
||||||
|
|
||||||
// Valid reports whether this set can cover at least one row.
|
|
||||||
valid() bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// RowList is a sequence of row keys.
|
|
||||||
type RowList []string
|
|
||||||
|
|
||||||
func (r RowList) proto() *btpb.RowSet {
|
|
||||||
keys := make([][]byte, len(r))
|
|
||||||
for i, row := range r {
|
|
||||||
keys[i] = []byte(row)
|
|
||||||
}
|
|
||||||
return &btpb.RowSet{RowKeys: keys}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowList) retainRowsAfter(lastRowKey string) RowSet {
|
|
||||||
var retryKeys RowList
|
|
||||||
for _, key := range r {
|
|
||||||
if key > lastRowKey {
|
|
||||||
retryKeys = append(retryKeys, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return retryKeys
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowList) valid() bool {
|
|
||||||
return len(r) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// A RowRange is a half-open interval [Start, Limit) encompassing
|
|
||||||
// all the rows with keys at least as large as Start, and less than Limit.
|
|
||||||
// (Bigtable string comparison is the same as Go's.)
|
|
||||||
// A RowRange can be unbounded, encompassing all keys at least as large as Start.
|
|
||||||
type RowRange struct {
|
|
||||||
start string
|
|
||||||
limit string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewRange returns the new RowRange [begin, end).
|
|
||||||
func NewRange(begin, end string) RowRange {
|
|
||||||
return RowRange{
|
|
||||||
start: begin,
|
|
||||||
limit: end,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unbounded tests whether a RowRange is unbounded.
|
|
||||||
func (r RowRange) Unbounded() bool {
|
|
||||||
return r.limit == ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Contains says whether the RowRange contains the key.
|
|
||||||
func (r RowRange) Contains(row string) bool {
|
|
||||||
return r.start <= row && (r.limit == "" || r.limit > row)
|
|
||||||
}
|
|
||||||
|
|
||||||
// String provides a printable description of a RowRange.
|
|
||||||
func (r RowRange) String() string {
|
|
||||||
a := strconv.Quote(r.start)
|
|
||||||
if r.Unbounded() {
|
|
||||||
return fmt.Sprintf("[%s,∞)", a)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("[%s,%q)", a, r.limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowRange) proto() *btpb.RowSet {
|
|
||||||
rr := &btpb.RowRange{
|
|
||||||
StartKey: &btpb.RowRange_StartKeyClosed{StartKeyClosed: []byte(r.start)},
|
|
||||||
}
|
|
||||||
if !r.Unbounded() {
|
|
||||||
rr.EndKey = &btpb.RowRange_EndKeyOpen{EndKeyOpen: []byte(r.limit)}
|
|
||||||
}
|
|
||||||
return &btpb.RowSet{RowRanges: []*btpb.RowRange{rr}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowRange) retainRowsAfter(lastRowKey string) RowSet {
|
|
||||||
if lastRowKey == "" || lastRowKey < r.start {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
// Set the beginning of the range to the row after the last scanned.
|
|
||||||
start := lastRowKey + "\x00"
|
|
||||||
if r.Unbounded() {
|
|
||||||
return InfiniteRange(start)
|
|
||||||
}
|
|
||||||
return NewRange(start, r.limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowRange) valid() bool {
|
|
||||||
return r.Unbounded() || r.start < r.limit
|
|
||||||
}
|
|
||||||
|
|
||||||
// RowRangeList is a sequence of RowRanges representing the union of the ranges.
|
|
||||||
type RowRangeList []RowRange
|
|
||||||
|
|
||||||
func (r RowRangeList) proto() *btpb.RowSet {
|
|
||||||
ranges := make([]*btpb.RowRange, len(r))
|
|
||||||
for i, rr := range r {
|
|
||||||
// RowRange.proto() returns a RowSet with a single element RowRange array
|
|
||||||
ranges[i] = rr.proto().RowRanges[0]
|
|
||||||
}
|
|
||||||
return &btpb.RowSet{RowRanges: ranges}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowRangeList) retainRowsAfter(lastRowKey string) RowSet {
|
|
||||||
if lastRowKey == "" {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
// Return a list of any range that has not yet been completely processed
|
|
||||||
var ranges RowRangeList
|
|
||||||
for _, rr := range r {
|
|
||||||
retained := rr.retainRowsAfter(lastRowKey)
|
|
||||||
if retained.valid() {
|
|
||||||
ranges = append(ranges, retained.(RowRange))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ranges
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r RowRangeList) valid() bool {
|
|
||||||
for _, rr := range r {
|
|
||||||
if rr.valid() {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// SingleRow returns a RowSet for reading a single row.
|
|
||||||
func SingleRow(row string) RowSet {
|
|
||||||
return RowList{row}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrefixRange returns a RowRange consisting of all keys starting with the prefix.
|
|
||||||
func PrefixRange(prefix string) RowRange {
|
|
||||||
return RowRange{
|
|
||||||
start: prefix,
|
|
||||||
limit: prefixSuccessor(prefix),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// InfiniteRange returns the RowRange consisting of all keys at least as
|
|
||||||
// large as start.
|
|
||||||
func InfiniteRange(start string) RowRange {
|
|
||||||
return RowRange{
|
|
||||||
start: start,
|
|
||||||
limit: "",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixSuccessor returns the lexically smallest string greater than the
|
|
||||||
// prefix, if it exists, or "" otherwise. In either case, it is the string
|
|
||||||
// needed for the Limit of a RowRange.
|
|
||||||
func prefixSuccessor(prefix string) string {
|
|
||||||
if prefix == "" {
|
|
||||||
return "" // infinite range
|
|
||||||
}
|
|
||||||
n := len(prefix)
|
|
||||||
for n--; n >= 0 && prefix[n] == '\xff'; n-- {
|
|
||||||
}
|
|
||||||
if n == -1 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
ans := []byte(prefix[:n])
|
|
||||||
ans = append(ans, prefix[n]+1)
|
|
||||||
return string(ans)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A ReadOption is an optional argument to ReadRows.
|
|
||||||
type ReadOption interface {
|
|
||||||
set(req *btpb.ReadRowsRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RowFilter returns a ReadOption that applies f to the contents of read rows.
|
|
||||||
//
|
|
||||||
// If multiple RowFilters are provided, only the last is used. To combine filters,
|
|
||||||
// use ChainFilters or InterleaveFilters instead.
|
|
||||||
func RowFilter(f Filter) ReadOption { return rowFilter{f} }
|
|
||||||
|
|
||||||
type rowFilter struct{ f Filter }
|
|
||||||
|
|
||||||
func (rf rowFilter) set(req *btpb.ReadRowsRequest) { req.Filter = rf.f.proto() }
|
|
||||||
|
|
||||||
// LimitRows returns a ReadOption that will limit the number of rows to be read.
|
|
||||||
func LimitRows(limit int64) ReadOption { return limitRows{limit} }
|
|
||||||
|
|
||||||
type limitRows struct{ limit int64 }
|
|
||||||
|
|
||||||
func (lr limitRows) set(req *btpb.ReadRowsRequest) { req.RowsLimit = lr.limit }
|
|
||||||
|
|
||||||
// mutationsAreRetryable returns true if all mutations are idempotent
|
|
||||||
// and therefore retryable. A mutation is idempotent iff all cell timestamps
|
|
||||||
// have an explicit timestamp set and do not rely on the timestamp being set on the server.
|
|
||||||
func mutationsAreRetryable(muts []*btpb.Mutation) bool {
|
|
||||||
serverTime := int64(ServerTime)
|
|
||||||
for _, mut := range muts {
|
|
||||||
setCell := mut.GetSetCell()
|
|
||||||
if setCell != nil && setCell.TimestampMicros == serverTime {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply applies a Mutation to a specific row.
|
|
||||||
func (t *Table) Apply(ctx context.Context, row string, m *Mutation, opts ...ApplyOption) error {
|
|
||||||
ctx = mergeOutgoingMetadata(ctx, t.md)
|
|
||||||
after := func(res proto.Message) {
|
|
||||||
for _, o := range opts {
|
|
||||||
o.after(res)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
ctx = traceStartSpan(ctx, "cloud.google.com/go/bigtable/Apply")
|
|
||||||
defer func() { traceEndSpan(ctx, err) }()
|
|
||||||
var callOptions []gax.CallOption
|
|
||||||
if m.cond == nil {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
RowKey: []byte(row),
|
|
||||||
Mutations: m.ops,
|
|
||||||
}
|
|
||||||
if mutationsAreRetryable(m.ops) {
|
|
||||||
callOptions = retryOptions
|
|
||||||
}
|
|
||||||
var res *btpb.MutateRowResponse
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context) error {
|
|
||||||
var err error
|
|
||||||
res, err = t.c.client.MutateRow(ctx, req)
|
|
||||||
return err
|
|
||||||
}, callOptions...)
|
|
||||||
if err == nil {
|
|
||||||
after(res)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &btpb.CheckAndMutateRowRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
RowKey: []byte(row),
|
|
||||||
PredicateFilter: m.cond.proto(),
|
|
||||||
}
|
|
||||||
if m.mtrue != nil {
|
|
||||||
if m.mtrue.cond != nil {
|
|
||||||
return errors.New("bigtable: conditional mutations cannot be nested")
|
|
||||||
}
|
|
||||||
req.TrueMutations = m.mtrue.ops
|
|
||||||
}
|
|
||||||
if m.mfalse != nil {
|
|
||||||
if m.mfalse.cond != nil {
|
|
||||||
return errors.New("bigtable: conditional mutations cannot be nested")
|
|
||||||
}
|
|
||||||
req.FalseMutations = m.mfalse.ops
|
|
||||||
}
|
|
||||||
if mutationsAreRetryable(req.TrueMutations) && mutationsAreRetryable(req.FalseMutations) {
|
|
||||||
callOptions = retryOptions
|
|
||||||
}
|
|
||||||
var cmRes *btpb.CheckAndMutateRowResponse
|
|
||||||
err = gax.Invoke(ctx, func(ctx context.Context) error {
|
|
||||||
var err error
|
|
||||||
cmRes, err = t.c.client.CheckAndMutateRow(ctx, req)
|
|
||||||
return err
|
|
||||||
}, callOptions...)
|
|
||||||
if err == nil {
|
|
||||||
after(cmRes)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// An ApplyOption is an optional argument to Apply.
|
|
||||||
type ApplyOption interface {
|
|
||||||
after(res proto.Message)
|
|
||||||
}
|
|
||||||
|
|
||||||
type applyAfterFunc func(res proto.Message)
|
|
||||||
|
|
||||||
func (a applyAfterFunc) after(res proto.Message) { a(res) }
|
|
||||||
|
|
||||||
// GetCondMutationResult returns an ApplyOption that reports whether the conditional
|
|
||||||
// mutation's condition matched.
|
|
||||||
func GetCondMutationResult(matched *bool) ApplyOption {
|
|
||||||
return applyAfterFunc(func(res proto.Message) {
|
|
||||||
if res, ok := res.(*btpb.CheckAndMutateRowResponse); ok {
|
|
||||||
*matched = res.PredicateMatched
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mutation represents a set of changes for a single row of a table.
|
|
||||||
type Mutation struct {
|
|
||||||
ops []*btpb.Mutation
|
|
||||||
|
|
||||||
// for conditional mutations
|
|
||||||
cond Filter
|
|
||||||
mtrue, mfalse *Mutation
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewMutation returns a new mutation.
|
|
||||||
func NewMutation() *Mutation {
|
|
||||||
return new(Mutation)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCondMutation returns a conditional mutation.
|
|
||||||
// The given row filter determines which mutation is applied:
|
|
||||||
// If the filter matches any cell in the row, mtrue is applied;
|
|
||||||
// otherwise, mfalse is applied.
|
|
||||||
// Either given mutation may be nil.
|
|
||||||
func NewCondMutation(cond Filter, mtrue, mfalse *Mutation) *Mutation {
|
|
||||||
return &Mutation{cond: cond, mtrue: mtrue, mfalse: mfalse}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set sets a value in a specified column, with the given timestamp.
|
|
||||||
// The timestamp will be truncated to millisecond granularity.
|
|
||||||
// A timestamp of ServerTime means to use the server timestamp.
|
|
||||||
func (m *Mutation) Set(family, column string, ts Timestamp, value []byte) {
|
|
||||||
m.ops = append(m.ops, &btpb.Mutation{Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: family,
|
|
||||||
ColumnQualifier: []byte(column),
|
|
||||||
TimestampMicros: int64(ts.TruncateToMilliseconds()),
|
|
||||||
Value: value,
|
|
||||||
}}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteCellsInColumn will delete all the cells whose columns are family:column.
|
|
||||||
func (m *Mutation) DeleteCellsInColumn(family, column string) {
|
|
||||||
m.ops = append(m.ops, &btpb.Mutation{Mutation: &btpb.Mutation_DeleteFromColumn_{DeleteFromColumn: &btpb.Mutation_DeleteFromColumn{
|
|
||||||
FamilyName: family,
|
|
||||||
ColumnQualifier: []byte(column),
|
|
||||||
}}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteTimestampRange deletes all cells whose columns are family:column
|
|
||||||
// and whose timestamps are in the half-open interval [start, end).
|
|
||||||
// If end is zero, it will be interpreted as infinity.
|
|
||||||
// The timestamps will be truncated to millisecond granularity.
|
|
||||||
func (m *Mutation) DeleteTimestampRange(family, column string, start, end Timestamp) {
|
|
||||||
m.ops = append(m.ops, &btpb.Mutation{Mutation: &btpb.Mutation_DeleteFromColumn_{DeleteFromColumn: &btpb.Mutation_DeleteFromColumn{
|
|
||||||
FamilyName: family,
|
|
||||||
ColumnQualifier: []byte(column),
|
|
||||||
TimeRange: &btpb.TimestampRange{
|
|
||||||
StartTimestampMicros: int64(start.TruncateToMilliseconds()),
|
|
||||||
EndTimestampMicros: int64(end.TruncateToMilliseconds()),
|
|
||||||
},
|
|
||||||
}}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteCellsInFamily will delete all the cells whose columns are family:*.
|
|
||||||
func (m *Mutation) DeleteCellsInFamily(family string) {
|
|
||||||
m.ops = append(m.ops, &btpb.Mutation{Mutation: &btpb.Mutation_DeleteFromFamily_{DeleteFromFamily: &btpb.Mutation_DeleteFromFamily{
|
|
||||||
FamilyName: family,
|
|
||||||
}}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteRow deletes the entire row.
|
|
||||||
func (m *Mutation) DeleteRow() {
|
|
||||||
m.ops = append(m.ops, &btpb.Mutation{Mutation: &btpb.Mutation_DeleteFromRow_{DeleteFromRow: &btpb.Mutation_DeleteFromRow{}}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// entryErr is a container that combines an entry with the error that was returned for it.
|
|
||||||
// Err may be nil if no error was returned for the Entry, or if the Entry has not yet been processed.
|
|
||||||
type entryErr struct {
|
|
||||||
Entry *btpb.MutateRowsRequest_Entry
|
|
||||||
Err error
|
|
||||||
}
|
|
||||||
|
|
||||||
// ApplyBulk applies multiple Mutations, up to a maximum of 100,000.
|
|
||||||
// Each mutation is individually applied atomically,
|
|
||||||
// but the set of mutations may be applied in any order.
|
|
||||||
//
|
|
||||||
// Two types of failures may occur. If the entire process
|
|
||||||
// fails, (nil, err) will be returned. If specific mutations
|
|
||||||
// fail to apply, ([]err, nil) will be returned, and the errors
|
|
||||||
// will correspond to the relevant rowKeys/muts arguments.
|
|
||||||
//
|
|
||||||
// Conditional mutations cannot be applied in bulk and providing one will result in an error.
|
|
||||||
func (t *Table) ApplyBulk(ctx context.Context, rowKeys []string, muts []*Mutation, opts ...ApplyOption) ([]error, error) {
|
|
||||||
ctx = mergeOutgoingMetadata(ctx, t.md)
|
|
||||||
if len(rowKeys) != len(muts) {
|
|
||||||
return nil, fmt.Errorf("mismatched rowKeys and mutation array lengths: %d, %d", len(rowKeys), len(muts))
|
|
||||||
}
|
|
||||||
|
|
||||||
origEntries := make([]*entryErr, len(rowKeys))
|
|
||||||
for i, key := range rowKeys {
|
|
||||||
mut := muts[i]
|
|
||||||
if mut.cond != nil {
|
|
||||||
return nil, errors.New("conditional mutations cannot be applied in bulk")
|
|
||||||
}
|
|
||||||
origEntries[i] = &entryErr{Entry: &btpb.MutateRowsRequest_Entry{RowKey: []byte(key), Mutations: mut.ops}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// entries will be reduced after each invocation to just what needs to be retried.
|
|
||||||
entries := make([]*entryErr, len(rowKeys))
|
|
||||||
copy(entries, origEntries)
|
|
||||||
var err error
|
|
||||||
ctx = traceStartSpan(ctx, "cloud.google.com/go/bigtable/ApplyBulk")
|
|
||||||
defer func() { traceEndSpan(ctx, err) }()
|
|
||||||
attrMap := make(map[string]interface{})
|
|
||||||
err = gax.Invoke(ctx, func(ctx context.Context) error {
|
|
||||||
attrMap["rowCount"] = len(entries)
|
|
||||||
tracePrintf(ctx, attrMap, "Row count in ApplyBulk")
|
|
||||||
err := t.doApplyBulk(ctx, entries, opts...)
|
|
||||||
if err != nil {
|
|
||||||
// We want to retry the entire request with the current entries
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
entries = t.getApplyBulkRetries(entries)
|
|
||||||
if len(entries) > 0 && len(idempotentRetryCodes) > 0 {
|
|
||||||
// We have at least one mutation that needs to be retried.
|
|
||||||
// Return an arbitrary error that is retryable according to callOptions.
|
|
||||||
return status.Errorf(idempotentRetryCodes[0], "Synthetic error: partial failure of ApplyBulk")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}, retryOptions...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Accumulate all of the errors into an array to return, interspersed with nils for successful
|
|
||||||
// entries. The absence of any errors means we should return nil.
|
|
||||||
var errs []error
|
|
||||||
var foundErr bool
|
|
||||||
for _, entry := range origEntries {
|
|
||||||
if entry.Err != nil {
|
|
||||||
foundErr = true
|
|
||||||
}
|
|
||||||
errs = append(errs, entry.Err)
|
|
||||||
}
|
|
||||||
if foundErr {
|
|
||||||
return errs, nil
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// getApplyBulkRetries returns the entries that need to be retried
|
|
||||||
func (t *Table) getApplyBulkRetries(entries []*entryErr) []*entryErr {
|
|
||||||
var retryEntries []*entryErr
|
|
||||||
for _, entry := range entries {
|
|
||||||
err := entry.Err
|
|
||||||
if err != nil && isIdempotentRetryCode[grpc.Code(err)] && mutationsAreRetryable(entry.Entry.Mutations) {
|
|
||||||
// There was an error and the entry is retryable.
|
|
||||||
retryEntries = append(retryEntries, entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return retryEntries
|
|
||||||
}
|
|
||||||
|
|
||||||
// doApplyBulk does the work of a single ApplyBulk invocation
|
|
||||||
func (t *Table) doApplyBulk(ctx context.Context, entryErrs []*entryErr, opts ...ApplyOption) error {
|
|
||||||
after := func(res proto.Message) {
|
|
||||||
for _, o := range opts {
|
|
||||||
o.after(res)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := make([]*btpb.MutateRowsRequest_Entry, len(entryErrs))
|
|
||||||
for i, entryErr := range entryErrs {
|
|
||||||
entries[i] = entryErr.Entry
|
|
||||||
}
|
|
||||||
req := &btpb.MutateRowsRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
Entries: entries,
|
|
||||||
}
|
|
||||||
stream, err := t.c.client.MutateRows(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
res, err := stream.Recv()
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, entry := range res.Entries {
|
|
||||||
s := entry.Status
|
|
||||||
if s.Code == int32(codes.OK) {
|
|
||||||
entryErrs[i].Err = nil
|
|
||||||
} else {
|
|
||||||
entryErrs[i].Err = status.Errorf(codes.Code(s.Code), s.Message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
after(res)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Timestamp is in units of microseconds since 1 January 1970.
|
|
||||||
type Timestamp int64
|
|
||||||
|
|
||||||
// ServerTime is a specific Timestamp that may be passed to (*Mutation).Set.
|
|
||||||
// It indicates that the server's timestamp should be used.
|
|
||||||
const ServerTime Timestamp = -1
|
|
||||||
|
|
||||||
// Time converts a time.Time into a Timestamp.
|
|
||||||
func Time(t time.Time) Timestamp { return Timestamp(t.UnixNano() / 1e3) }
|
|
||||||
|
|
||||||
// Now returns the Timestamp representation of the current time on the client.
|
|
||||||
func Now() Timestamp { return Time(time.Now()) }
|
|
||||||
|
|
||||||
// Time converts a Timestamp into a time.Time.
|
|
||||||
func (ts Timestamp) Time() time.Time { return time.Unix(0, int64(ts)*1e3) }
|
|
||||||
|
|
||||||
// TruncateToMilliseconds truncates a Timestamp to millisecond granularity,
|
|
||||||
// which is currently the only granularity supported.
|
|
||||||
func (ts Timestamp) TruncateToMilliseconds() Timestamp {
|
|
||||||
if ts == ServerTime {
|
|
||||||
return ts
|
|
||||||
}
|
|
||||||
return ts - ts%1000
|
|
||||||
}
|
|
||||||
|
|
||||||
// ApplyReadModifyWrite applies a ReadModifyWrite to a specific row.
|
|
||||||
// It returns the newly written cells.
|
|
||||||
func (t *Table) ApplyReadModifyWrite(ctx context.Context, row string, m *ReadModifyWrite) (Row, error) {
|
|
||||||
ctx = mergeOutgoingMetadata(ctx, t.md)
|
|
||||||
req := &btpb.ReadModifyWriteRowRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
RowKey: []byte(row),
|
|
||||||
Rules: m.ops,
|
|
||||||
}
|
|
||||||
res, err := t.c.client.ReadModifyWriteRow(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if res.Row == nil {
|
|
||||||
return nil, errors.New("unable to apply ReadModifyWrite: res.Row=nil")
|
|
||||||
}
|
|
||||||
r := make(Row)
|
|
||||||
for _, fam := range res.Row.Families { // res is *btpb.Row, fam is *btpb.Family
|
|
||||||
decodeFamilyProto(r, row, fam)
|
|
||||||
}
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadModifyWrite represents a set of operations on a single row of a table.
|
|
||||||
// It is like Mutation but for non-idempotent changes.
|
|
||||||
// When applied, these operations operate on the latest values of the row's cells,
|
|
||||||
// and result in a new value being written to the relevant cell with a timestamp
|
|
||||||
// that is max(existing timestamp, current server time).
|
|
||||||
//
|
|
||||||
// The application of a ReadModifyWrite is atomic; concurrent ReadModifyWrites will
|
|
||||||
// be executed serially by the server.
|
|
||||||
type ReadModifyWrite struct {
|
|
||||||
ops []*btpb.ReadModifyWriteRule
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewReadModifyWrite returns a new ReadModifyWrite.
|
|
||||||
func NewReadModifyWrite() *ReadModifyWrite { return new(ReadModifyWrite) }
|
|
||||||
|
|
||||||
// AppendValue appends a value to a specific cell's value.
|
|
||||||
// If the cell is unset, it will be treated as an empty value.
|
|
||||||
func (m *ReadModifyWrite) AppendValue(family, column string, v []byte) {
|
|
||||||
m.ops = append(m.ops, &btpb.ReadModifyWriteRule{
|
|
||||||
FamilyName: family,
|
|
||||||
ColumnQualifier: []byte(column),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_AppendValue{AppendValue: v},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Increment interprets the value in a specific cell as a 64-bit big-endian signed integer,
|
|
||||||
// and adds a value to it. If the cell is unset, it will be treated as zero.
|
|
||||||
// If the cell is set and is not an 8-byte value, the entire ApplyReadModifyWrite
|
|
||||||
// operation will fail.
|
|
||||||
func (m *ReadModifyWrite) Increment(family, column string, delta int64) {
|
|
||||||
m.ops = append(m.ops, &btpb.ReadModifyWriteRule{
|
|
||||||
FamilyName: family,
|
|
||||||
ColumnQualifier: []byte(column),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_IncrementAmount{IncrementAmount: delta},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// mergeOutgoingMetadata returns a context populated by the existing outgoing metadata,
|
|
||||||
// if any, joined with internal metadata.
|
|
||||||
func mergeOutgoingMetadata(ctx context.Context, md metadata.MD) context.Context {
|
|
||||||
mdCopy, _ := metadata.FromOutgoingContext(ctx)
|
|
||||||
return metadata.NewOutgoingContext(ctx, metadata.Join(mdCopy, md))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *Table) SampleRowKeys(ctx context.Context) ([]string, error) {
|
|
||||||
ctx = mergeOutgoingMetadata(ctx, t.md)
|
|
||||||
var sampledRowKeys []string
|
|
||||||
err := gax.Invoke(ctx, func(ctx context.Context) error {
|
|
||||||
sampledRowKeys = nil
|
|
||||||
req := &btpb.SampleRowKeysRequest{
|
|
||||||
TableName: t.c.fullTableName(t.table),
|
|
||||||
AppProfileId: t.c.appProfile,
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(ctx) // for aborting the stream
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
stream, err := t.c.client.SampleRowKeys(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
res, err := stream.Recv()
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
key := string(res.RowKey)
|
|
||||||
if key == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
sampledRowKeys = append(sampledRowKeys, key)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}, retryOptions...)
|
|
||||||
return sampledRowKeys, err
|
|
||||||
}
|
|
||||||
1159
vendor/cloud.google.com/go/bigtable/bigtable_test.go
generated
vendored
1159
vendor/cloud.google.com/go/bigtable/bigtable_test.go
generated
vendored
File diff suppressed because it is too large
Load Diff
83
vendor/cloud.google.com/go/bigtable/bttest/example_test.go
generated
vendored
83
vendor/cloud.google.com/go/bigtable/bttest/example_test.go
generated
vendored
@ -1,83 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2016 Google LLC
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
package bttest_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigtable"
|
|
||||||
"cloud.google.com/go/bigtable/bttest"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
"google.golang.org/api/option"
|
|
||||||
"google.golang.org/grpc"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ExampleNewServer() {
|
|
||||||
|
|
||||||
srv, err := bttest.NewServer("localhost:0")
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
proj, instance := "proj", "instance"
|
|
||||||
|
|
||||||
adminClient, err := bigtable.NewAdminClient(ctx, proj, instance, option.WithGRPCConn(conn))
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = adminClient.CreateTable(ctx, "example"); err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = adminClient.CreateColumnFamily(ctx, "example", "links"); err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := bigtable.NewClient(ctx, proj, instance, option.WithGRPCConn(conn))
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
tbl := client.Open("example")
|
|
||||||
|
|
||||||
mut := bigtable.NewMutation()
|
|
||||||
mut.Set("links", "golang.org", bigtable.Now(), []byte("Gophers!"))
|
|
||||||
if err = tbl.Apply(ctx, "com.google.cloud", mut); err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if row, err := tbl.ReadRow(ctx, "com.google.cloud"); err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
} else {
|
|
||||||
for _, column := range row["links"] {
|
|
||||||
fmt.Println(column.Column)
|
|
||||||
fmt.Println(string(column.Value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output:
|
|
||||||
// links:golang.org
|
|
||||||
// Gophers!
|
|
||||||
}
|
|
||||||
1331
vendor/cloud.google.com/go/bigtable/bttest/inmem.go
generated
vendored
1331
vendor/cloud.google.com/go/bigtable/bttest/inmem.go
generated
vendored
File diff suppressed because it is too large
Load Diff
805
vendor/cloud.google.com/go/bigtable/bttest/inmem_test.go
generated
vendored
805
vendor/cloud.google.com/go/bigtable/bttest/inmem_test.go
generated
vendored
@ -1,805 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package bttest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
"github.com/google/go-cmp/cmp/cmpopts"
|
|
||||||
"golang.org/x/net/context"
|
|
||||||
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
|
|
||||||
btpb "google.golang.org/genproto/googleapis/bigtable/v2"
|
|
||||||
"google.golang.org/grpc"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestConcurrentMutationsReadModifyAndGC(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
if _, err := s.CreateTable(
|
|
||||||
ctx,
|
|
||||||
&btapb.CreateTableRequest{Parent: "cluster", TableId: "t"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
const name = `cluster/tables/t`
|
|
||||||
tbl := s.tables[name]
|
|
||||||
req := &btapb.ModifyColumnFamiliesRequest{
|
|
||||||
Name: name,
|
|
||||||
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
||||||
Id: "cf",
|
|
||||||
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{Create: &btapb.ColumnFamily{}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
_, err := s.ModifyColumnFamilies(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req = &btapb.ModifyColumnFamiliesRequest{
|
|
||||||
Name: name,
|
|
||||||
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
||||||
Id: "cf",
|
|
||||||
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{Update: &btapb.ColumnFamily{
|
|
||||||
GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}},
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.ModifyColumnFamilies(ctx, req); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
var ts int64
|
|
||||||
ms := func() []*btpb.Mutation {
|
|
||||||
return []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte(`col`),
|
|
||||||
TimestampMicros: atomic.AddInt64(&ts, 1000),
|
|
||||||
}},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
rmw := func() *btpb.ReadModifyWriteRowRequest {
|
|
||||||
return &btpb.ReadModifyWriteRowRequest{
|
|
||||||
TableName: name,
|
|
||||||
RowKey: []byte(fmt.Sprint(rand.Intn(100))),
|
|
||||||
Rules: []*btpb.ReadModifyWriteRule{{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("col"),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_IncrementAmount{IncrementAmount: 1},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for ctx.Err() == nil {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: name,
|
|
||||||
RowKey: []byte(fmt.Sprint(rand.Intn(100))),
|
|
||||||
Mutations: ms(),
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, req); err != nil {
|
|
||||||
panic(err) // can't use t.Fatal in goroutine
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for ctx.Err() == nil {
|
|
||||||
_, _ = s.ReadModifyWriteRow(ctx, rmw())
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
tbl.gc()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
wg.Wait()
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(1 * time.Second):
|
|
||||||
t.Error("Concurrent mutations and GCs haven't completed after 1s")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCreateTableWithFamily(t *testing.T) {
|
|
||||||
// The Go client currently doesn't support creating a table with column families
|
|
||||||
// in one operation but it is allowed by the API. This must still be supported by the
|
|
||||||
// fake server so this test lives here instead of in the main bigtable
|
|
||||||
// integration test.
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf1": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 123}}},
|
|
||||||
"cf2": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 456}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
cTbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
tbl, err := s.GetTable(ctx, &btapb.GetTableRequest{Name: cTbl.Name})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Getting table: %v", err)
|
|
||||||
}
|
|
||||||
cf := tbl.ColumnFamilies["cf1"]
|
|
||||||
if cf == nil {
|
|
||||||
t.Fatalf("Missing col family cf1")
|
|
||||||
}
|
|
||||||
if got, want := cf.GcRule.GetMaxNumVersions(), int32(123); got != want {
|
|
||||||
t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
|
|
||||||
}
|
|
||||||
cf = tbl.ColumnFamilies["cf2"]
|
|
||||||
if cf == nil {
|
|
||||||
t.Fatalf("Missing col family cf2")
|
|
||||||
}
|
|
||||||
if got, want := cf.GcRule.GetMaxNumVersions(), int32(456); got != want {
|
|
||||||
t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type MockSampleRowKeysServer struct {
|
|
||||||
responses []*btpb.SampleRowKeysResponse
|
|
||||||
grpc.ServerStream
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MockSampleRowKeysServer) Send(resp *btpb.SampleRowKeysResponse) error {
|
|
||||||
s.responses = append(s.responses, resp)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSampleRowKeys(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Populate the table
|
|
||||||
val := []byte("value")
|
|
||||||
rowCount := 1000
|
|
||||||
for i := 0; i < rowCount; i++ {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: tbl.Name,
|
|
||||||
RowKey: []byte("row-" + strconv.Itoa(i)),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("col"),
|
|
||||||
TimestampMicros: 1000,
|
|
||||||
Value: val,
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Populating table: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mock := &MockSampleRowKeysServer{}
|
|
||||||
if err := s.SampleRowKeys(&btpb.SampleRowKeysRequest{TableName: tbl.Name}, mock); err != nil {
|
|
||||||
t.Errorf("SampleRowKeys error: %v", err)
|
|
||||||
}
|
|
||||||
if len(mock.responses) == 0 {
|
|
||||||
t.Fatal("Response count: got 0, want > 0")
|
|
||||||
}
|
|
||||||
// Make sure the offset of the final response is the offset of the final row
|
|
||||||
got := mock.responses[len(mock.responses)-1].OffsetBytes
|
|
||||||
want := int64((rowCount - 1) * len(val))
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Invalid offset: got %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDropRowRange(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tbl := s.tables[tblInfo.Name]
|
|
||||||
|
|
||||||
// Populate the table
|
|
||||||
prefixes := []string{"AAA", "BBB", "CCC", "DDD"}
|
|
||||||
count := 3
|
|
||||||
doWrite := func() {
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
RowKey: []byte(prefix + strconv.Itoa(i)),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("col"),
|
|
||||||
TimestampMicros: 1000,
|
|
||||||
Value: []byte{},
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Populating table: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
doWrite()
|
|
||||||
tblSize := tbl.rows.Len()
|
|
||||||
req := &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_RowKeyPrefix{RowKeyPrefix: []byte("AAA")},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping first range: %v", err)
|
|
||||||
}
|
|
||||||
got, want := tbl.rows.Len(), tblSize-count
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after first drop: got %d (%v), want %d", got, tbl.rows, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
req = &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_RowKeyPrefix{RowKeyPrefix: []byte("DDD")},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping second range: %v", err)
|
|
||||||
}
|
|
||||||
got, want = tbl.rows.Len(), tblSize-(2*count)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after second drop: got %d (%v), want %d", got, tbl.rows, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
req = &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_RowKeyPrefix{RowKeyPrefix: []byte("XXX")},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping invalid range: %v", err)
|
|
||||||
}
|
|
||||||
got, want = tbl.rows.Len(), tblSize-(2*count)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after invalid drop: got %d (%v), want %d", got, tbl.rows, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
req = &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_DeleteAllDataFromTable{DeleteAllDataFromTable: true},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping all data: %v", err)
|
|
||||||
}
|
|
||||||
got, want = tbl.rows.Len(), 0
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after drop all: got %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that we can write rows, delete some and then write them again.
|
|
||||||
count = 1
|
|
||||||
doWrite()
|
|
||||||
|
|
||||||
req = &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_DeleteAllDataFromTable{DeleteAllDataFromTable: true},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping all data: %v", err)
|
|
||||||
}
|
|
||||||
got, want = tbl.rows.Len(), 0
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after drop all: got %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
doWrite()
|
|
||||||
got, want = tbl.rows.Len(), len(prefixes)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after rewrite: got %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
req = &btapb.DropRowRangeRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Target: &btapb.DropRowRangeRequest_RowKeyPrefix{RowKeyPrefix: []byte("BBB")},
|
|
||||||
}
|
|
||||||
if _, err = s.DropRowRange(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Dropping range: %v", err)
|
|
||||||
}
|
|
||||||
doWrite()
|
|
||||||
got, want = tbl.rows.Len(), len(prefixes)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("Row count after drop range: got %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type MockReadRowsServer struct {
|
|
||||||
responses []*btpb.ReadRowsResponse
|
|
||||||
grpc.ServerStream
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *MockReadRowsServer) Send(resp *btpb.ReadRowsResponse) error {
|
|
||||||
s.responses = append(s.responses, resp)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRows(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf0": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
mreq := &btpb.MutateRowRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
RowKey: []byte("row"),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf0",
|
|
||||||
ColumnQualifier: []byte("col"),
|
|
||||||
TimestampMicros: 1000,
|
|
||||||
Value: []byte{},
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, mreq); err != nil {
|
|
||||||
t.Fatalf("Populating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rowset := range []*btpb.RowSet{
|
|
||||||
{RowKeys: [][]byte{[]byte("row")}},
|
|
||||||
{RowRanges: []*btpb.RowRange{{StartKey: &btpb.RowRange_StartKeyClosed{StartKeyClosed: []byte("")}}}},
|
|
||||||
{RowRanges: []*btpb.RowRange{{StartKey: &btpb.RowRange_StartKeyClosed{StartKeyClosed: []byte("r")}}}},
|
|
||||||
{RowRanges: []*btpb.RowRange{{
|
|
||||||
StartKey: &btpb.RowRange_StartKeyClosed{StartKeyClosed: []byte("")},
|
|
||||||
EndKey: &btpb.RowRange_EndKeyOpen{EndKeyOpen: []byte("s")},
|
|
||||||
}}},
|
|
||||||
} {
|
|
||||||
mock := &MockReadRowsServer{}
|
|
||||||
req := &btpb.ReadRowsRequest{TableName: tblInfo.Name, Rows: rowset}
|
|
||||||
if err = s.ReadRows(req, mock); err != nil {
|
|
||||||
t.Fatalf("ReadRows error: %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(mock.responses), 1; got != want {
|
|
||||||
t.Errorf("%+v: response count: got %d, want %d", rowset, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRowsOrder(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf0": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
count := 3
|
|
||||||
mcf := func(i int) *btapb.ModifyColumnFamiliesRequest {
|
|
||||||
return &btapb.ModifyColumnFamiliesRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
||||||
Id: "cf" + strconv.Itoa(i),
|
|
||||||
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{Create: &btapb.ColumnFamily{}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 1; i <= count; i++ {
|
|
||||||
_, err = s.ModifyColumnFamilies(ctx, mcf(i))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Populate the table
|
|
||||||
for fc := 0; fc < count; fc++ {
|
|
||||||
for cc := count; cc > 0; cc-- {
|
|
||||||
for tc := 0; tc < count; tc++ {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
RowKey: []byte("row"),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf" + strconv.Itoa(fc),
|
|
||||||
ColumnQualifier: []byte("col" + strconv.Itoa(cc)),
|
|
||||||
TimestampMicros: int64((tc + 1) * 1000),
|
|
||||||
Value: []byte{},
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, req); err != nil {
|
|
||||||
t.Fatalf("Populating table: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
req := &btpb.ReadRowsRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
Rows: &btpb.RowSet{RowKeys: [][]byte{[]byte("row")}},
|
|
||||||
}
|
|
||||||
mock := &MockReadRowsServer{}
|
|
||||||
if err = s.ReadRows(req, mock); err != nil {
|
|
||||||
t.Errorf("ReadRows error: %v", err)
|
|
||||||
}
|
|
||||||
if len(mock.responses) == 0 {
|
|
||||||
t.Fatal("Response count: got 0, want > 0")
|
|
||||||
}
|
|
||||||
if len(mock.responses[0].Chunks) != 27 {
|
|
||||||
t.Fatalf("Chunk count: got %d, want 27", len(mock.responses[0].Chunks))
|
|
||||||
}
|
|
||||||
testOrder := func(ms *MockReadRowsServer) {
|
|
||||||
var prevFam, prevCol string
|
|
||||||
var prevTime int64
|
|
||||||
for _, cc := range ms.responses[0].Chunks {
|
|
||||||
if prevFam == "" {
|
|
||||||
prevFam = cc.FamilyName.Value
|
|
||||||
prevCol = string(cc.Qualifier.Value)
|
|
||||||
prevTime = cc.TimestampMicros
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if cc.FamilyName.Value < prevFam {
|
|
||||||
t.Errorf("Family order is not correct: got %s < %s", cc.FamilyName.Value, prevFam)
|
|
||||||
} else if cc.FamilyName.Value == prevFam {
|
|
||||||
if string(cc.Qualifier.Value) < prevCol {
|
|
||||||
t.Errorf("Column order is not correct: got %s < %s", string(cc.Qualifier.Value), prevCol)
|
|
||||||
} else if string(cc.Qualifier.Value) == prevCol {
|
|
||||||
if cc.TimestampMicros > prevTime {
|
|
||||||
t.Errorf("cell order is not correct: got %d > %d", cc.TimestampMicros, prevTime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
prevFam = cc.FamilyName.Value
|
|
||||||
prevCol = string(cc.Qualifier.Value)
|
|
||||||
prevTime = cc.TimestampMicros
|
|
||||||
}
|
|
||||||
}
|
|
||||||
testOrder(mock)
|
|
||||||
|
|
||||||
// Read with interleave filter
|
|
||||||
inter := &btpb.RowFilter_Interleave{}
|
|
||||||
fnr := &btpb.RowFilter{Filter: &btpb.RowFilter_FamilyNameRegexFilter{FamilyNameRegexFilter: "1"}}
|
|
||||||
cqr := &btpb.RowFilter{Filter: &btpb.RowFilter_ColumnQualifierRegexFilter{ColumnQualifierRegexFilter: []byte("2")}}
|
|
||||||
inter.Filters = append(inter.Filters, fnr, cqr)
|
|
||||||
req = &btpb.ReadRowsRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
Rows: &btpb.RowSet{RowKeys: [][]byte{[]byte("row")}},
|
|
||||||
Filter: &btpb.RowFilter{
|
|
||||||
Filter: &btpb.RowFilter_Interleave_{Interleave: inter},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
mock = &MockReadRowsServer{}
|
|
||||||
if err = s.ReadRows(req, mock); err != nil {
|
|
||||||
t.Errorf("ReadRows error: %v", err)
|
|
||||||
}
|
|
||||||
if len(mock.responses) == 0 {
|
|
||||||
t.Fatal("Response count: got 0, want > 0")
|
|
||||||
}
|
|
||||||
if len(mock.responses[0].Chunks) != 18 {
|
|
||||||
t.Fatalf("Chunk count: got %d, want 18", len(mock.responses[0].Chunks))
|
|
||||||
}
|
|
||||||
testOrder(mock)
|
|
||||||
|
|
||||||
// Check order after ReadModifyWriteRow
|
|
||||||
rmw := func(i int) *btpb.ReadModifyWriteRowRequest {
|
|
||||||
return &btpb.ReadModifyWriteRowRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
RowKey: []byte("row"),
|
|
||||||
Rules: []*btpb.ReadModifyWriteRule{{
|
|
||||||
FamilyName: "cf3",
|
|
||||||
ColumnQualifier: []byte("col" + strconv.Itoa(i)),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_IncrementAmount{IncrementAmount: 1},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := count; i > 0; i-- {
|
|
||||||
if _, err := s.ReadModifyWriteRow(ctx, rmw(i)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
req = &btpb.ReadRowsRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
Rows: &btpb.RowSet{RowKeys: [][]byte{[]byte("row")}},
|
|
||||||
}
|
|
||||||
mock = &MockReadRowsServer{}
|
|
||||||
if err = s.ReadRows(req, mock); err != nil {
|
|
||||||
t.Errorf("ReadRows error: %v", err)
|
|
||||||
}
|
|
||||||
if len(mock.responses) == 0 {
|
|
||||||
t.Fatal("Response count: got 0, want > 0")
|
|
||||||
}
|
|
||||||
if len(mock.responses[0].Chunks) != 30 {
|
|
||||||
t.Fatalf("Chunk count: got %d, want 30", len(mock.responses[0].Chunks))
|
|
||||||
}
|
|
||||||
testOrder(mock)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckAndMutateRowWithoutPredicate(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Populate the table
|
|
||||||
val := []byte("value")
|
|
||||||
mrreq := &btpb.MutateRowRequest{
|
|
||||||
TableName: tbl.Name,
|
|
||||||
RowKey: []byte("row-present"),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("col"),
|
|
||||||
TimestampMicros: 1000,
|
|
||||||
Value: val,
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, mrreq); err != nil {
|
|
||||||
t.Fatalf("Populating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &btpb.CheckAndMutateRowRequest{
|
|
||||||
TableName: tbl.Name,
|
|
||||||
RowKey: []byte("row-not-present"),
|
|
||||||
}
|
|
||||||
if res, err := s.CheckAndMutateRow(ctx, req); err != nil {
|
|
||||||
t.Errorf("CheckAndMutateRow error: %v", err)
|
|
||||||
} else if got, want := res.PredicateMatched, false; got != want {
|
|
||||||
t.Errorf("Invalid PredicateMatched value: got %t, want %t", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
req = &btpb.CheckAndMutateRowRequest{
|
|
||||||
TableName: tbl.Name,
|
|
||||||
RowKey: []byte("row-present"),
|
|
||||||
}
|
|
||||||
if res, err := s.CheckAndMutateRow(ctx, req); err != nil {
|
|
||||||
t.Errorf("CheckAndMutateRow error: %v", err)
|
|
||||||
} else if got, want := res.PredicateMatched, true; got != want {
|
|
||||||
t.Errorf("Invalid PredicateMatched value: got %t, want %t", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServer_ReadModifyWriteRow(t *testing.T) {
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{MaxNumVersions: 1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Creating table: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &btpb.ReadModifyWriteRowRequest{
|
|
||||||
TableName: tbl.Name,
|
|
||||||
RowKey: []byte("row-key"),
|
|
||||||
Rules: []*btpb.ReadModifyWriteRule{
|
|
||||||
{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("q1"),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_AppendValue{
|
|
||||||
AppendValue: []byte("a"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
// multiple ops for same cell
|
|
||||||
{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("q1"),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_AppendValue{
|
|
||||||
AppendValue: []byte("b"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
// different cell whose qualifier should sort before the prior rules
|
|
||||||
{
|
|
||||||
FamilyName: "cf",
|
|
||||||
ColumnQualifier: []byte("q0"),
|
|
||||||
Rule: &btpb.ReadModifyWriteRule_IncrementAmount{
|
|
||||||
IncrementAmount: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := s.ReadModifyWriteRow(ctx, req)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadModifyWriteRow error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := &btpb.ReadModifyWriteRowResponse{
|
|
||||||
Row: &btpb.Row{
|
|
||||||
Key: []byte("row-key"),
|
|
||||||
Families: []*btpb.Family{{
|
|
||||||
Name: "cf",
|
|
||||||
Columns: []*btpb.Column{
|
|
||||||
{
|
|
||||||
Qualifier: []byte("q0"),
|
|
||||||
Cells: []*btpb.Cell{{
|
|
||||||
Value: []byte{0, 0, 0, 0, 0, 0, 0, 1},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Qualifier: []byte("q1"),
|
|
||||||
Cells: []*btpb.Cell{{
|
|
||||||
Value: []byte("ab"),
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
diff := cmp.Diff(got, want, cmpopts.IgnoreFields(btpb.Cell{}, "TimestampMicros"))
|
|
||||||
if diff != "" {
|
|
||||||
t.Errorf("unexpected response: %s", diff)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// helper function to populate table data
|
|
||||||
func populateTable(ctx context.Context, s *server) (*btapb.Table, error) {
|
|
||||||
newTbl := btapb.Table{
|
|
||||||
ColumnFamilies: map[string]*btapb.ColumnFamily{
|
|
||||||
"cf0": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
count := 3
|
|
||||||
mcf := func(i int) *btapb.ModifyColumnFamiliesRequest {
|
|
||||||
return &btapb.ModifyColumnFamiliesRequest{
|
|
||||||
Name: tblInfo.Name,
|
|
||||||
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
||||||
Id: "cf" + strconv.Itoa(i),
|
|
||||||
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 1; i <= count; i++ {
|
|
||||||
_, err = s.ModifyColumnFamilies(ctx, mcf(i))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Populate the table
|
|
||||||
for fc := 0; fc < count; fc++ {
|
|
||||||
for cc := count; cc > 0; cc-- {
|
|
||||||
for tc := 0; tc < count; tc++ {
|
|
||||||
req := &btpb.MutateRowRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
RowKey: []byte("row"),
|
|
||||||
Mutations: []*btpb.Mutation{{
|
|
||||||
Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
|
|
||||||
FamilyName: "cf" + strconv.Itoa(fc),
|
|
||||||
ColumnQualifier: []byte("col" + strconv.Itoa(cc)),
|
|
||||||
TimestampMicros: int64((tc + 1) * 1000),
|
|
||||||
Value: []byte{},
|
|
||||||
}},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if _, err := s.MutateRow(ctx, req); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return tblInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilters(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
in *btpb.RowFilter
|
|
||||||
out int
|
|
||||||
}{
|
|
||||||
{in: &btpb.RowFilter{Filter: &btpb.RowFilter_BlockAllFilter{true}}, out: 0},
|
|
||||||
{in: &btpb.RowFilter{Filter: &btpb.RowFilter_BlockAllFilter{false}}, out: 1},
|
|
||||||
{in: &btpb.RowFilter{Filter: &btpb.RowFilter_PassAllFilter{true}}, out: 1},
|
|
||||||
{in: &btpb.RowFilter{Filter: &btpb.RowFilter_PassAllFilter{false}}, out: 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
s := &server{
|
|
||||||
tables: make(map[string]*table),
|
|
||||||
}
|
|
||||||
|
|
||||||
tblInfo, err := populateTable(ctx, s)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req := &btpb.ReadRowsRequest{
|
|
||||||
TableName: tblInfo.Name,
|
|
||||||
Rows: &btpb.RowSet{RowKeys: [][]byte{[]byte("row")}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
req.Filter = tc.in
|
|
||||||
|
|
||||||
mock := &MockReadRowsServer{}
|
|
||||||
if err = s.ReadRows(req, mock); err != nil {
|
|
||||||
t.Errorf("ReadRows error: %v", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(mock.responses) != tc.out {
|
|
||||||
t.Errorf("Response count: got %d, want %d", len(mock.responses), tc.out)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
1425
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbt.go
generated
vendored
1425
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbt.go
generated
vendored
File diff suppressed because it is too large
Load Diff
224
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbt_test.go
generated
vendored
224
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbt_test.go
generated
vendored
@ -1,224 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigtable"
|
|
||||||
"cloud.google.com/go/internal/testutil"
|
|
||||||
"github.com/google/go-cmp/cmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseDuration(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
in string
|
|
||||||
// out or fail are mutually exclusive
|
|
||||||
out time.Duration
|
|
||||||
fail bool
|
|
||||||
}{
|
|
||||||
{in: "10ms", out: 10 * time.Millisecond},
|
|
||||||
{in: "3s", out: 3 * time.Second},
|
|
||||||
{in: "60m", out: 60 * time.Minute},
|
|
||||||
{in: "12h", out: 12 * time.Hour},
|
|
||||||
{in: "7d", out: 168 * time.Hour},
|
|
||||||
|
|
||||||
{in: "", fail: true},
|
|
||||||
{in: "0", fail: true},
|
|
||||||
{in: "7ns", fail: true},
|
|
||||||
{in: "14mo", fail: true},
|
|
||||||
{in: "3.5h", fail: true},
|
|
||||||
{in: "106752d", fail: true}, // overflow
|
|
||||||
}
|
|
||||||
for _, tc := range tests {
|
|
||||||
got, err := parseDuration(tc.in)
|
|
||||||
if !tc.fail && err != nil {
|
|
||||||
t.Errorf("parseDuration(%q) unexpectedly failed: %v", tc.in, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail && err == nil {
|
|
||||||
t.Errorf("parseDuration(%q) did not fail", tc.in)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if got != tc.out {
|
|
||||||
t.Errorf("parseDuration(%q) = %v, want %v", tc.in, got, tc.out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGCPolicy(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
in string
|
|
||||||
out bigtable.GCPolicy
|
|
||||||
fail bool
|
|
||||||
}{
|
|
||||||
{in: "maxage=1h", out: bigtable.MaxAgePolicy(time.Hour * 1)},
|
|
||||||
{in: "maxversions=2", out: bigtable.MaxVersionsPolicy(int(2))},
|
|
||||||
{in: "maxversions=2 and maxage=1h", out: bigtable.IntersectionPolicy([]bigtable.GCPolicy{bigtable.MaxVersionsPolicy(int(2)), bigtable.MaxAgePolicy(time.Hour * 1)}...)},
|
|
||||||
{in: "maxversions=2 or maxage=1h", out: bigtable.UnionPolicy([]bigtable.GCPolicy{bigtable.MaxVersionsPolicy(int(2)), bigtable.MaxAgePolicy(time.Hour * 1)}...)},
|
|
||||||
|
|
||||||
{in: "maxage=1", fail: true},
|
|
||||||
{in: "maxage = 1h", fail: true},
|
|
||||||
{in: "maxage =1h", fail: true},
|
|
||||||
{in: "maxage= 1h", fail: true},
|
|
||||||
{in: "foomaxage=1h", fail: true},
|
|
||||||
{in: "maxversions=1h", fail: true},
|
|
||||||
{in: "maxversions= 1", fail: true},
|
|
||||||
{in: "maxversions = 1", fail: true},
|
|
||||||
{in: "maxversions =1", fail: true},
|
|
||||||
{in: "barmaxversions=1", fail: true},
|
|
||||||
{in: "maxage = 1h or maxversions=1h", fail: true},
|
|
||||||
{in: "foomaxversions=2 or maxage=1h", fail: true},
|
|
||||||
{in: "maxversions=2 or barmaxage=1h", fail: true},
|
|
||||||
{in: "foomaxversions=2 or barmaxage=1h", fail: true},
|
|
||||||
{in: "maxage = 1h and maxversions=1h", fail: true},
|
|
||||||
{in: "foomaxage=1h and maxversions=1", fail: true},
|
|
||||||
{in: "maxage=1h and barmaxversions=1", fail: true},
|
|
||||||
{in: "foomaxage=1h and barmaxversions=1", fail: true},
|
|
||||||
}
|
|
||||||
for _, tc := range tests {
|
|
||||||
got, err := parseGCPolicy(tc.in)
|
|
||||||
if !tc.fail && err != nil {
|
|
||||||
t.Errorf("parseGCPolicy(%q) unexpectedly failed: %v", tc.in, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail && err == nil {
|
|
||||||
t.Errorf("parseGCPolicy(%q) did not fail", tc.in)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var cmpOpts cmp.Options
|
|
||||||
cmpOpts = append(cmpOpts, cmp.AllowUnexported(bigtable.IntersectionPolicy([]bigtable.GCPolicy{}...)), cmp.AllowUnexported(bigtable.UnionPolicy([]bigtable.GCPolicy{}...)))
|
|
||||||
if !cmp.Equal(got, tc.out, cmpOpts) {
|
|
||||||
t.Errorf("parseGCPolicy(%q) =%v, want %v", tc.in, got, tc.out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseArgs(t *testing.T) {
|
|
||||||
got, err := parseArgs([]string{"a=1", "b=2"}, []string{"a", "b"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := map[string]string{"a": "1", "b": "2"}
|
|
||||||
if !testutil.Equal(got, want) {
|
|
||||||
t.Fatalf("got %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := parseArgs([]string{"a1"}, []string{"a1"}); err == nil {
|
|
||||||
t.Error("malformed: got nil, want error")
|
|
||||||
}
|
|
||||||
if _, err := parseArgs([]string{"a=1"}, []string{"b"}); err == nil {
|
|
||||||
t.Error("invalid: got nil, want error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseColumnsFilter(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
in string
|
|
||||||
out bigtable.Filter
|
|
||||||
fail bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
in: "columnA",
|
|
||||||
out: bigtable.ColumnFilter("columnA"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA:columnA",
|
|
||||||
out: bigtable.ChainFilters(bigtable.FamilyFilter("familyA"), bigtable.ColumnFilter("columnA")),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "columnA,columnB",
|
|
||||||
out: bigtable.InterleaveFilters(bigtable.ColumnFilter("columnA"), bigtable.ColumnFilter("columnB")),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA:columnA,columnB",
|
|
||||||
out: bigtable.InterleaveFilters(
|
|
||||||
bigtable.ChainFilters(bigtable.FamilyFilter("familyA"), bigtable.ColumnFilter("columnA")),
|
|
||||||
bigtable.ColumnFilter("columnB"),
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "columnA,familyB:columnB",
|
|
||||||
out: bigtable.InterleaveFilters(
|
|
||||||
bigtable.ColumnFilter("columnA"),
|
|
||||||
bigtable.ChainFilters(bigtable.FamilyFilter("familyB"), bigtable.ColumnFilter("columnB")),
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA:columnA,familyB:columnB",
|
|
||||||
out: bigtable.InterleaveFilters(
|
|
||||||
bigtable.ChainFilters(bigtable.FamilyFilter("familyA"), bigtable.ColumnFilter("columnA")),
|
|
||||||
bigtable.ChainFilters(bigtable.FamilyFilter("familyB"), bigtable.ColumnFilter("columnB")),
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA:",
|
|
||||||
out: bigtable.FamilyFilter("familyA"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: ":columnA",
|
|
||||||
out: bigtable.ColumnFilter("columnA"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: ",:columnA,,familyB:columnB,",
|
|
||||||
out: bigtable.InterleaveFilters(
|
|
||||||
bigtable.ColumnFilter("columnA"),
|
|
||||||
bigtable.ChainFilters(bigtable.FamilyFilter("familyB"), bigtable.ColumnFilter("columnB")),
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA:columnA:cellA",
|
|
||||||
fail: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
in: "familyA::columnA",
|
|
||||||
fail: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
got, err := parseColumnsFilter(tc.in)
|
|
||||||
|
|
||||||
if !tc.fail && err != nil {
|
|
||||||
t.Errorf("parseColumnsFilter(%q) unexpectedly failed: %v", tc.in, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail && err == nil {
|
|
||||||
t.Errorf("parseColumnsFilter(%q) did not fail", tc.in)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if tc.fail {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var cmpOpts cmp.Options
|
|
||||||
cmpOpts =
|
|
||||||
append(
|
|
||||||
cmpOpts,
|
|
||||||
cmp.AllowUnexported(bigtable.ChainFilters([]bigtable.Filter{}...)),
|
|
||||||
cmp.AllowUnexported(bigtable.InterleaveFilters([]bigtable.Filter{}...)))
|
|
||||||
|
|
||||||
if !cmp.Equal(got, tc.out, cmpOpts) {
|
|
||||||
t.Errorf("parseColumnsFilter(%q) = %v, want %v", tc.in, got, tc.out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
369
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbtdoc.go
generated
vendored
369
vendor/cloud.google.com/go/bigtable/cmd/cbt/cbtdoc.go
generated
vendored
@ -1,369 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
// DO NOT EDIT. THIS IS AUTOMATICALLY GENERATED.
|
|
||||||
// Run "go generate" to regenerate.
|
|
||||||
//go:generate go run cbt.go -o cbtdoc.go doc
|
|
||||||
|
|
||||||
/*
|
|
||||||
Cbt is a tool for doing basic interactions with Cloud Bigtable. To learn how to
|
|
||||||
install the cbt tool, see the
|
|
||||||
[cbt overview](https://cloud.google.com/bigtable/docs/go/cbt-overview).
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
|
|
||||||
cbt [options] command [arguments]
|
|
||||||
|
|
||||||
The commands are:
|
|
||||||
|
|
||||||
count Count rows in a table
|
|
||||||
createinstance Create an instance with an initial cluster
|
|
||||||
createcluster Create a cluster in the configured instance (replication alpha)
|
|
||||||
createfamily Create a column family
|
|
||||||
createtable Create a table
|
|
||||||
updatecluster Update a cluster in the configured instance
|
|
||||||
deleteinstance Deletes an instance
|
|
||||||
deletecluster Deletes a cluster from the configured instance (replication alpha)
|
|
||||||
deletecolumn Delete all cells in a column
|
|
||||||
deletefamily Delete a column family
|
|
||||||
deleterow Delete a row
|
|
||||||
deletetable Delete a table
|
|
||||||
doc Print godoc-suitable documentation for cbt
|
|
||||||
help Print help text
|
|
||||||
listinstances List instances in a project
|
|
||||||
listclusters List instances in an instance
|
|
||||||
lookup Read from a single row
|
|
||||||
ls List tables and column families
|
|
||||||
mddoc Print documentation for cbt in Markdown format
|
|
||||||
read Read rows
|
|
||||||
set Set value of a cell
|
|
||||||
setgcpolicy Set the GC policy for a column family
|
|
||||||
waitforreplication Blocks until all the completed writes have been replicated to all the clusters (replication alpha)
|
|
||||||
createtablefromsnapshot Create a table from a snapshot (snapshots alpha)
|
|
||||||
createsnapshot Create a snapshot from a source table (snapshots alpha)
|
|
||||||
listsnapshots List snapshots in a cluster (snapshots alpha)
|
|
||||||
getsnapshot Get snapshot info (snapshots alpha)
|
|
||||||
deletesnapshot Delete snapshot in a cluster (snapshots alpha)
|
|
||||||
version Print the current cbt version
|
|
||||||
|
|
||||||
Use "cbt help <command>" for more information about a command.
|
|
||||||
|
|
||||||
The options are:
|
|
||||||
|
|
||||||
-project string
|
|
||||||
project ID, if unset uses gcloud configured project
|
|
||||||
-instance string
|
|
||||||
Cloud Bigtable instance
|
|
||||||
-creds string
|
|
||||||
if set, use application credentials in this file
|
|
||||||
|
|
||||||
|
|
||||||
Alpha features are not currently available to most Cloud Bigtable customers. The
|
|
||||||
features might be changed in backward-incompatible ways and are not recommended
|
|
||||||
for production use. They are not subject to any SLA or deprecation policy.
|
|
||||||
|
|
||||||
For convenience, values of the -project, -instance, -creds,
|
|
||||||
-admin-endpoint and -data-endpoint flags may be specified in
|
|
||||||
~/.cbtrc in this format:
|
|
||||||
project = my-project-123
|
|
||||||
instance = my-instance
|
|
||||||
creds = path-to-account-key.json
|
|
||||||
admin-endpoint = hostname:port
|
|
||||||
data-endpoint = hostname:port
|
|
||||||
All values are optional, and all will be overridden by flags.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Count rows in a table
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt count <table>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create an instance with an initial cluster
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createinstance <instance-id> <display-name> <cluster-id> <zone> <num-nodes> <storage type>
|
|
||||||
instance-id Permanent, unique id for the instance
|
|
||||||
display-name Description of the instance
|
|
||||||
cluster-id Permanent, unique id for the cluster in the instance
|
|
||||||
zone The zone in which to create the cluster
|
|
||||||
num-nodes The number of nodes to create
|
|
||||||
storage-type SSD or HDD
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create a cluster in the configured instance (replication alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createcluster <cluster-id> <zone> <num-nodes> <storage type>
|
|
||||||
cluster-id Permanent, unique id for the cluster in the instance
|
|
||||||
zone The zone in which to create the cluster
|
|
||||||
num-nodes The number of nodes to create
|
|
||||||
storage-type SSD or HDD
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create a column family
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createfamily <table> <family>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create a table
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createtable <table> [families=family[:(maxage=<d> | maxversions=<n>)],...] [splits=split,...]
|
|
||||||
families: Column families and their associated GC policies. See "setgcpolicy".
|
|
||||||
Example: families=family1:maxage=1w,family2:maxversions=1
|
|
||||||
splits: Row key to be used to initially split the table
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Update a cluster in the configured instance
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt updatecluster <cluster-id> [num-nodes=num-nodes]
|
|
||||||
cluster-id Permanent, unique id for the cluster in the instance
|
|
||||||
num-nodes The number of nodes to update to
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Deletes an instance
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deleteinstance <instance>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Deletes a cluster from the configured instance (replication alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deletecluster <cluster>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Delete all cells in a column
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deletecolumn <table> <row> <family> <column> [app-profile=<app profile id>]
|
|
||||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Delete a column family
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deletefamily <table> <family>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Delete a row
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deleterow <table> <row> [app-profile=<app profile id>]
|
|
||||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Delete a table
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deletetable <table>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Print godoc-suitable documentation for cbt
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt doc
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Print help text
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt help [command]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
List instances in a project
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt listinstances
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
List instances in an instance
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt listclusters
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Read from a single row
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt lookup <table> <row> [cells-per-column=<n>] [app-profile=<app profile id>]
|
|
||||||
cells-per-column=<n> Read only this many cells per column
|
|
||||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
List tables and column families
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt ls List tables
|
|
||||||
cbt ls <table> List column families in <table>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Print documentation for cbt in Markdown format
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt mddoc
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Read rows
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt read <table> [start=<row>] [end=<row>] [prefix=<prefix>] [regex=<regex>] [count=<n>] [cells-per-column=<n>] [app-profile=<app profile id>]
|
|
||||||
start=<row> Start reading at this row
|
|
||||||
end=<row> Stop reading before this row
|
|
||||||
prefix=<prefix> Read rows with this prefix
|
|
||||||
regex=<regex> Read rows with keys matching this regex
|
|
||||||
count=<n> Read only this many rows
|
|
||||||
cells-per-column=<n> Read only this many cells per column
|
|
||||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Set value of a cell
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt set <table> <row> [app-profile=<app profile id>] family:column=val[@ts] ...
|
|
||||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
|
||||||
family:column=val[@ts] may be repeated to set multiple cells.
|
|
||||||
|
|
||||||
ts is an optional integer timestamp.
|
|
||||||
If it cannot be parsed, the `@ts` part will be
|
|
||||||
interpreted as part of the value.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Set the GC policy for a column family
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt setgcpolicy <table> <family> ( maxage=<d> | maxversions=<n> )
|
|
||||||
|
|
||||||
maxage=<d> Maximum timestamp age to preserve (e.g. "1h", "4d")
|
|
||||||
maxversions=<n> Maximum number of versions to preserve
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Blocks until all the completed writes have been replicated to all the clusters (replication alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt waitforreplication <table>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create a table from a snapshot (snapshots alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createtablefromsnapshot <table> <cluster> <snapshot>
|
|
||||||
table The name of the table to create
|
|
||||||
cluster The cluster where the snapshot is located
|
|
||||||
snapshot The snapshot to restore
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Create a snapshot from a source table (snapshots alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt createsnapshot <cluster> <snapshot> <table> [ttl=<d>]
|
|
||||||
|
|
||||||
[ttl=<d>] Lifespan of the snapshot (e.g. "1h", "4d")
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
List snapshots in a cluster (snapshots alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt listsnapshots [<cluster>]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Get snapshot info (snapshots alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt getsnapshot <cluster> <snapshot>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Delete snapshot in a cluster (snapshots alpha)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt deletesnapshot <cluster> <snapshot>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Print the current cbt version
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
cbt version
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
*/
|
|
||||||
package main
|
|
||||||
44
vendor/cloud.google.com/go/bigtable/cmd/emulator/cbtemulator.go
generated
vendored
44
vendor/cloud.google.com/go/bigtable/cmd/emulator/cbtemulator.go
generated
vendored
@ -1,44 +0,0 @@
|
|||||||
// Copyright 2016 Google LLC
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
/*
|
|
||||||
cbtemulator launches the in-memory Cloud Bigtable server on the given address.
|
|
||||||
*/
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
|
|
||||||
"cloud.google.com/go/bigtable/bttest"
|
|
||||||
"google.golang.org/grpc"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
host = flag.String("host", "localhost", "the address to bind to on the local machine")
|
|
||||||
port = flag.Int("port", 9000, "the port number to bind to on the local machine")
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
grpc.EnableTracing = false
|
|
||||||
flag.Parse()
|
|
||||||
srv, err := bttest.NewServer(fmt.Sprintf("%s:%d", *host, *port))
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to start emulator: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("Cloud Bigtable emulator running on %s\n", srv.Addr)
|
|
||||||
select {}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user