Package org.apache.beam.sdk.extensions.gcp.util

Class GcsUtil

java.lang.Object

org.apache.beam.sdk.extensions.gcp.util.GcsUtil

public class GcsUtil
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	GcsUtil.CreateOptions
static class	GcsUtil.GcsCountersOptions
static class	GcsUtil.GcsUtilFactory
static class	GcsUtil.StorageObjectOrIOException

Method Summary

Modifier and Type	Method	Description
boolean	bucketAccessible(GcsPath path)
long	bucketOwner(GcsPath path)
void	copy(Iterable<String> srcFilenames, Iterable<String> destFilenames)
void	copy(Iterable<GcsPath> srcPaths, Iterable<GcsPath> dstPaths, org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.OverwriteStrategy strategy)	experimental api.
void	copyV2(Iterable<GcsPath> srcPaths, Iterable<GcsPath> dstPaths)	experimental api.
WritableByteChannel	create(GcsPath path, String type)	Deprecated. Use create(GcsPath, CreateOptions) instead.
WritableByteChannel	create(GcsPath path, String type, Integer uploadBufferSizeBytes)	Deprecated. Use create(GcsPath, CreateOptions) instead.
WritableByteChannel	create(GcsPath path, GcsUtil.CreateOptions options)
void	createBucket(com.google.cloud.storage.BucketInfo bucketInfo)
void	createBucket(String projectId, Bucket bucket)	Deprecated. use createBucket(BucketInfo).
List<GcsPath>	expand(GcsPath gcsPattern)
long	fileSize(GcsPath path)
com.google.cloud.storage.Blob	getBlob(GcsPath gcsPath, com.google.cloud.storage.Storage.BlobGetOption... options)
List<org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.BlobResult>	getBlobs(Iterable<GcsPath> gcsPaths, com.google.cloud.storage.Storage.BlobGetOption... options)
@Nullable Bucket	getBucket(GcsPath path)	Deprecated. use getBucketWithOptions(GcsPath, BucketGetOption...) .
@Nullable com.google.cloud.storage.Bucket	getBucketWithOptions(GcsPath path, com.google.cloud.storage.Storage.BucketGetOption... options)
static String	getNonWildcardPrefix(String globExp)
StorageObject	getObject(GcsPath gcsPath)	Deprecated. use invalid reference #getBlob(GcsPath) .
List<GcsUtil.StorageObjectOrIOException>	getObjects(List<GcsPath> gcsPaths)	Deprecated. use getBlobs(Iterable, BlobGetOption...).
static boolean	isWildcard(GcsPath spec)
com.google.api.gax.paging.Page<com.google.cloud.storage.Blob>	listBlobs(String bucket, String prefix, @Nullable String pageToken, com.google.cloud.storage.Storage.BlobListOption... options)
com.google.api.gax.paging.Page<com.google.cloud.storage.Blob>	listBlobs(String bucket, String prefix, @Nullable String pageToken, @Nullable String delimiter, com.google.cloud.storage.Storage.BlobListOption... options)
Objects	listObjects(String bucket, String prefix, @Nullable String pageToken)	Deprecated. use listBlobs(String, String, String, BlobListOption...).
Objects	listObjects(String bucket, String prefix, @Nullable String pageToken, @Nullable String delimiter)	Deprecated. use listBlobs(String, String, String, String, BlobListOption...).
SeekableByteChannel	open(GcsPath path)
void	remove(Iterable<GcsPath> paths, org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.MissingStrategy strategy)	experimental api.
void	remove(Collection<String> filenames)
void	removeBucket(Bucket bucket)	Deprecated. use removeBucket(BucketInfo).
void	removeBucket(com.google.cloud.storage.BucketInfo bucketInfo)
void	removeV2(Iterable<GcsPath> paths)	experimental api.
void	rename(Iterable<String> srcFilenames, Iterable<String> destFilenames, MoveOptions... moveOptions)
void	rename(Iterable<GcsPath> srcPaths, Iterable<GcsPath> dstPaths, org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.MissingStrategy srcMissing, org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.OverwriteStrategy dstOverwrite)	experimental api.
void	renameV2(Iterable<GcsPath> srcPaths, Iterable<GcsPath> dstPaths, MoveOptions... moveOptions)	experimental api.
protected void	setBatchRequestSupplier(Supplier<org.apache.beam.sdk.extensions.gcp.util.GcsUtilV1.BatchInterface> supplier)
protected void	setStorageClient(Storage storageClient)
void	verifyBucketAccessible(GcsPath path)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Details

getNonWildcardPrefix

public static String getNonWildcardPrefix(String globExp)

isWildcard

public static boolean isWildcard(GcsPath spec)

setStorageClient

protected void setStorageClient(Storage storageClient)

setBatchRequestSupplier

protected void setBatchRequestSupplier(Supplier<org.apache.beam.sdk.extensions.gcp.util.GcsUtilV1.BatchInterface> supplier)

expand

public List<GcsPath> expand(GcsPath gcsPattern)
                     throws IOException

Throws:

IOException

fileSize

public long fileSize(GcsPath path)
              throws IOException

Throws:

IOException

getObject

@Deprecated
public StorageObject getObject(GcsPath gcsPath)
                        throws IOException

Deprecated.

use

invalid reference

#getBlob(GcsPath)

.

Throws:

IOException

getBlob

public com.google.cloud.storage.Blob getBlob(GcsPath gcsPath,
 com.google.cloud.storage.Storage.BlobGetOption... options)
                                      throws IOException

Throws:

IOException

getObjects

@Deprecated
public List<GcsUtil.StorageObjectOrIOException> getObjects(List<GcsPath> gcsPaths)
                                                    throws IOException

Deprecated.

use getBlobs(Iterable, BlobGetOption...).

Throws:

IOException

getBlobs

public List<org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.BlobResult> getBlobs(Iterable<GcsPath> gcsPaths,
 com.google.cloud.storage.Storage.BlobGetOption... options)
                                                                            throws IOException

Throws:

IOException

listObjects

@Deprecated
public Objects listObjects(String bucket,
 String prefix,
 @Nullable String pageToken)
                    throws IOException

Deprecated.

use listBlobs(String, String, String, BlobListOption...).

Throws:

IOException

listObjects

@Deprecated
public Objects listObjects(String bucket,
 String prefix,
 @Nullable String pageToken,
 @Nullable String delimiter)
                    throws IOException

Deprecated.

use listBlobs(String, String, String, String, BlobListOption...).

Throws:

IOException

listBlobs

public com.google.api.gax.paging.Page<com.google.cloud.storage.Blob> listBlobs(String bucket,
 String prefix,
 @Nullable String pageToken,
 com.google.cloud.storage.Storage.BlobListOption... options)
                                                                        throws IOException

Throws:

IOException

listBlobs

public com.google.api.gax.paging.Page<com.google.cloud.storage.Blob> listBlobs(String bucket,
 String prefix,
 @Nullable String pageToken,
 @Nullable String delimiter,
 com.google.cloud.storage.Storage.BlobListOption... options)
                                                                        throws IOException

Throws:

IOException

open

public SeekableByteChannel open(GcsPath path)
                         throws IOException

Throws:

IOException

create

@Deprecated
public WritableByteChannel create(GcsPath path,
 String type)
                           throws IOException

Deprecated.

Use create(GcsPath, CreateOptions) instead.

Throws:

IOException

create

@Deprecated
public WritableByteChannel create(GcsPath path,
 String type,
 Integer uploadBufferSizeBytes)
                           throws IOException

Deprecated.

Use create(GcsPath, CreateOptions) instead.

Throws:

IOException

create

public WritableByteChannel create(GcsPath path,
 GcsUtil.CreateOptions options)
                           throws IOException

Throws:

IOException

verifyBucketAccessible

public void verifyBucketAccessible(GcsPath path)
                            throws IOException

Throws:

IOException

bucketAccessible

public boolean bucketAccessible(GcsPath path)
                         throws IOException

Throws:

IOException

bucketOwner

public long bucketOwner(GcsPath path)
                 throws IOException

Throws:

IOException

createBucket

@Deprecated
public void createBucket(String projectId,
 Bucket bucket)
                  throws IOException

Deprecated.

use createBucket(BucketInfo).

Throws:

IOException

createBucket

public void createBucket(com.google.cloud.storage.BucketInfo bucketInfo)
                  throws IOException

Throws:

IOException

getBucket

@Deprecated
public @Nullable Bucket getBucket(GcsPath path)
                           throws IOException

Deprecated.

use getBucketWithOptions(GcsPath, BucketGetOption...) .

Throws:

IOException

getBucketWithOptions

public @Nullable com.google.cloud.storage.Bucket getBucketWithOptions(GcsPath path,
 com.google.cloud.storage.Storage.BucketGetOption... options)
                                                               throws IOException

Throws:

IOException

removeBucket

@Deprecated
public void removeBucket(Bucket bucket)
                  throws IOException

Deprecated.

use removeBucket(BucketInfo).

Throws:

IOException

removeBucket

public void removeBucket(com.google.cloud.storage.BucketInfo bucketInfo)
                  throws IOException

Throws:

IOException

copy

public void copy(Iterable<String> srcFilenames,
 Iterable<String> destFilenames)
          throws IOException

Throws:

IOException

copyV2

public void copyV2(Iterable<GcsPath> srcPaths,
 Iterable<GcsPath> dstPaths)
            throws IOException

experimental api.

Throws:

IOException

copy

public void copy(Iterable<GcsPath> srcPaths,
 Iterable<GcsPath> dstPaths,
 org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.OverwriteStrategy strategy)
          throws IOException

experimental api.

Throws:

IOException

rename

public void rename(Iterable<String> srcFilenames,
 Iterable<String> destFilenames,
 MoveOptions... moveOptions)
            throws IOException

Throws:

IOException

renameV2

public void renameV2(Iterable<GcsPath> srcPaths,
 Iterable<GcsPath> dstPaths,
 MoveOptions... moveOptions)
              throws IOException

experimental api.

Throws:

IOException

rename

public void rename(Iterable<GcsPath> srcPaths,
 Iterable<GcsPath> dstPaths,
 org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.MissingStrategy srcMissing,
 org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.OverwriteStrategy dstOverwrite)
            throws IOException

experimental api.

Throws:

IOException

remove

public void remove(Collection<String> filenames)
            throws IOException

Throws:

IOException

removeV2

public void removeV2(Iterable<GcsPath> paths)
              throws IOException

experimental api.

Throws:

IOException

remove

public void remove(Iterable<GcsPath> paths,
 org.apache.beam.sdk.extensions.gcp.util.GcsUtilV2.MissingStrategy strategy)
            throws IOException

experimental api.

Throws:

IOException