How to download files from S3 to data science server?


#1

I have a file in S3. I see the file name when I do

print(c3.S3.listFiles(urlOrEncodedPath=s3_path)) # there is a file here

Now I want to download it to my data science server,

c3.s3_download(s3_path = s3_path, local_path = local_path)

This just prints the following in python, without downloading:

s3_download({'s3_path': 'XXX', 'local_path': 'XXX', 'type': 's3_download'})

The server version is 7.8.8


#2

import shutil
try:
from http.client import HTTPSConnection, HTTPConnection
except ImportError:
from httplib import HTTPSConnection, HTTPConnection

def open_remote_file(srcUrlOrEncodedPath):
url = ‘/file/1/{tenant}/{tag}/{src_url}’.format(
tenant=c3.connection._tenant,
tag=c3.connection._tag,
src_url=srcUrlOrEncodedPath
)
Connection = (HTTPSConnection if c3.connection._c3_server_parsed_url.scheme == ‘https’ else HTTPConnection)
conn = Connection(c3.connection._c3_server_parsed_url.netloc)
conn.request(url=url, method=‘GET’, headers=c3.connection._headers)
c3.connection._set_keep_alive_on_socket(conn.sock)
return conn.getresponse()

def download_remote_file(srcUrlOrEncodedPath, local_dest_path):
fsrc = open_remote_file(srcUrlOrEncodedPath)
fdst = open(local_dest_path,‘wb’)
shutil.copyfileobj(fsrc,fdst)