Well, no answers here, but after a lot of experimentation I figured it out. Some resumable errors throw an exception but others do not. A solution is to check for a non-200 exit code for errors that don't throw exceptions and then throw an exception directly, and handle both cases in the except: section.
response = requests.post(session_url, data=data, headers=headers)
if response.status_code != 200:
raise RuntimeError(f"non-200 response {response.status_code}")
Here's my full code that others might find useful.
import os
import pickle
import requests
import argparse
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import mimetypes
import sys
DryRun = False
Debug = False
def printstr(s):
print(s, end='')
sys.stdout.flush()
def authenticate():
creds = None
SCOPES = ['https://www.googleapis.com/auth/photoslibrary.sharing']
os.makedirs(os.path.expanduser('~/.credentials/gphotos'), exist_ok=True)
CLIENT_SECRET_FILE = os.path.expanduser('~/.credentials/gphotos/client_secret.json')
TOKEN_FILE = os.path.expanduser('~/.credentials/gphotos/token.pickle')
if os.path.exists(TOKEN_FILE):
with open(TOKEN_FILE, 'rb') as token:
creds = pickle.load(token)
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
creds = flow.run_local_server(port=0)
with open(TOKEN_FILE, 'wb') as token:
pickle.dump(creds, token)
return creds
def create_album(creds, album_name, collab):
if DryRun:
print(f"Dry run created album {album_name}")
return { 'id':0, 'shareableURL': 'NO_URL' }
try:
headers = {
'Authorization': 'Bearer ' + creds.token,
}
json_body = {
"album": {
"title": album_name
}
}
response = (requests.post(
'https://photoslibrary.googleapis.com/v1/albums',
json=json_body, headers=headers)).json()
printstr(f"Created album {album_name} ... ")
albumID = response['id']
json_body = {
"sharedAlbumOptions": {
"isCollaborative": "true" if collab else "false",
"isCommentable": "true"
}
}
response = (requests.post(
'https://photoslibrary.googleapis.com/v1/albums/' + albumID + ':share',
json=json_body, headers=headers)).json()
print("sharing options set")
shareableURL = response['shareInfo']['shareableUrl']
return {'id':albumID, 'shareableURL': shareableURL}
except Exception as error:
print(f"An error occurred trying to create album {album_name}: {error}")
raise RuntimeError('Create album failed') from error
def is_media(file_path):
# Return mime type if is an image or video, otherwise False
(type, _) = mimetypes.guess_type(file_path)
if type and (str(type).startswith('image/') or str(type).startswith('video/')):
return type
else:
return False
def upload_photos(creds, folder_paths, collab, chunk_factor):
failures = []
albums = []
for folder_path in folder_paths:
if not os.path.isdir(folder_path):
print(f"Bad argument {folder_path}, it is not a folder")
continue
for root, dirs, files in os.walk(folder_path):
# sort dirs in place, so at next level they are visited alphabetically
dirs.sort()
# determine if root has media files and so will become an album
mediaFiles = [ f for f in files if is_media(f) ]
if len(mediaFiles)>0:
# create an album corresponding to root
mediaFiles.sort()
album_name = os.path.basename(root)
album_path = root
try:
album = create_album(creds, album_name, collab)
album_id = album['id']
albums.append([album_name,album['shareableURL']])
except Exception:
print(f"Continuing on to next album")
failures.append(('Album creation failure:',
album_name, 'from', album_path))
continue
# upload the media into the album
for file in mediaFiles:
file_path = os.path.join(album_path, file)
try:
upload_photo(creds, file_path, file, album_id, album_name, chunk_factor)
except Exception:
failures.append(['Media upload failure:', file, 'in',
album_name, 'from', file_path])
print("Continuing to next file")
print(f"Number of failures: {len(failures)}")
for failure in failures:
print(' '.join(failure))
print("HTML Index")
albums.sort(reverse=True)
for (name, url) in albums:
print(f'<p><a target="_blank" href="{url}">{name}</a></p>')
def upload_photo(creds, file_path, file_name, album_id, album_name, chunk_factor):
if DryRun:
print(f"Dry run upload {file_name}")
return
try:
file_size = os.path.getsize(file_path)
content_type = is_media(file_name)
printstr(f"Uploading {file_name} of type {content_type} of length {file_size} ... ")
# Create resumable session
headers = {
'Authorization': f'Bearer {creds.token}',
'X-Goog-Upload-Content-Type': content_type,
'X-Goog-Upload-Protocol': 'resumable',
'X-Goog-Upload-Command': 'start',
'X-Goog-Upload-Raw-Size': str(file_size)
}
response = requests.post('https://photoslibrary.googleapis.com/v1/uploads',
headers=headers)
session_url = response.headers['X-Goog-Upload-URL']
chunk_size = int(response.headers['X-Goog-Upload-Chunk-Granularity']) * chunk_factor
printstr(f"created session with chunk size {chunk_size} ... ")
offset = 0
final = False
# Perform the upload
exception_count = 0
with open(file_path, 'rb') as f:
chunk_number = 0
while not final:
chunk_number += 1
data_size = min(chunk_size, file_size - offset)
if offset + data_size >= file_size:
command = 'upload, finalize'
else:
command = 'upload'
f.seek(offset)
if Debug and chunk_number == 2:
# Force an unexpected EOF error on second chunk
data = f.read(data_size - 100)
else:
data = f.read(data_size)
headers = {
'Authorization': 'Bearer ' + creds.token,
'Content-Length': str(data_size),
'X-Goog-Upload-Content-Type': content_type,
'X-Goog-Upload-Command': command,
'X-Goog-Upload-Offset': str(offset)
}
try:
response = requests.post(session_url, data=data, headers=headers)
if response.status_code != 200:
raise RuntimeError(f"non-200 response {response.status_code}")
printstr(f"uploaded chunk {chunk_number} ... ")
offset += data_size
if offset >= file_size:
final = True
continue
except (RuntimeError, OSError, requests.exceptions.RequestException) as e:
printstr(f"error uploading chunk {chunk_number} ... ")
exception_count += 1
if exception_count > 4:
raise RuntimeError(f"Too many exceptions")
query_headers = {
'Authorization': 'Bearer ' + creds.token,
'Content-Length': "0",
'X-Goog-Upload-Command': 'query'
}
query_response = requests.post(session_url, headers=query_headers)
if query_response.status_code != 200:
raise RuntimeError(f"Query failed with code {response.status_code}")
offset = int(query_response.headers['X-Goog-Upload-Size-Received'])
printstr(f"resuming at {offset} ... ")
uploadToken = response.text
# add the uploaded media to the album
headers = {'Authorization': 'Bearer ' + creds.token}
json_body = {
'albumId':album_id,
'newMediaItems':[{
'description':"",
'simpleMediaItem':{'uploadToken':uploadToken}
}]
}
response = requests.post(
'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate',
json=json_body, headers=headers)
print(f"added to album {album_name}")
except Exception as error:
print(f"\nCould not upload file {file_name} due to error {error}")
raise RuntimeError('Media upload failed') from error
def main():
parser = argparse.ArgumentParser(description='Upload photo albums to Google Photos.')
parser.add_argument('-dryrun', action='store_true')
parser.add_argument('-collab', action='store_true')
parser.add_argument('-debug', action='store_true')
parser.add_argument('-chunk', type=int, default=20)
parser.add_argument('folderpaths', metavar='dirs', nargs='+',
type=os.path.abspath,
help='One or more folders containing photos and/or subfolders')
args = parser.parse_args()
global Debug
Debug = args.debug
global DryRun
DryRun = args.dryrun
if DryRun:
print(f"Dry run protocol")
creds = authenticate()
upload_photos(creds, args.folderpaths, args.collab, args.chunk )
if __name__ == '__main__':
main()
