Coverage for src/bioimageio/spec/_upload.py: 81%

1import collections.abc 

2import io 

3from typing import Union 

4from zipfile import ZipFile 

5 

6import httpx 

7from loguru import logger 

8 

9from ._description import InvalidDescr, ResourceDescr, build_description 

10from ._internal._settings import settings 

11from ._internal.common_nodes import ResourceDescrBase 

12from ._internal.io import BioimageioYamlContent, get_reader 

13from ._internal.io_basics import BIOIMAGEIO_YAML 

14from ._internal.io_utils import write_yaml 

15from ._internal.validation_context import get_validation_context 

16from ._io import load_description 

17from ._package import get_package_content 

18from .common import HttpUrl, PermissiveFileSource 

19 

20 

21# TODO: remove alpha stage warning 

22def upload( 

23 source: Union[PermissiveFileSource, ZipFile, ResourceDescr, BioimageioYamlContent], 

24 /, 

25 keep_remote_files_as_references: bool = False, 

26) -> HttpUrl: 

27 """Upload a new resource description (version) to the hypha server to be shared at bioimage.io. 

28 To edit an existing resource **version**, please login to https://bioimage.io and use the web interface. 

29 

30 WARNING: This upload function is in alpha stage and might change in the future. 

31 

32 Args: 

33 source: The resource description to upload. 

34 keep_remote_files_as_references: If True, remote files will be kept as references and not downloaded and uploaded to the server. 

35 

36 Returns: 

37 A URL to the uploaded resource description. 

38 Note: It might take some time until the resource is processed and available for download from the returned URL. 

39 """ 

40 

41 if settings.hypha_upload_token is None: 

42 raise ValueError( 

43 """ 

44Upload token is not set. Please set BIOIMAGEIO_HYPHA_UPLOAD_TOKEN in your environment variables. 

45By setting this token you agree to our terms of service at https://bioimage.io/#/toc. 

46 

47How to obtain a token: 

48 1. Login to https://bioimage.io 

49 2. Generate a new token at https://bioimage.io/#/api?tab=hypha-rpc 

50""" 

51 ) 

52 

53 if isinstance(source, ResourceDescrBase): 

54 # If source is already a ResourceDescr, we can use it directly 

55 descr = source 

56 elif isinstance(source, dict): 

57 descr = build_description(source) 

58 else: 

59 descr = load_description(source) 

60 

61 if isinstance(descr, InvalidDescr): 

62 raise ValueError("Uploading invalid resource descriptions is not allowed.") 

63 

64 if descr.type != "model": 

65 raise NotImplementedError( 

66 f"For now, only model resources can be uploaded (got type={descr.type})." 

67 ) 

68 

69 if descr.id is not None: 

70 raise ValueError( 

71 "You cannot upload a resource with an id. Please remove the id from the description and make sure to upload a new non-existing resource. To edit an existing resource, please use the web interface at https://bioimage.io." 

72 ) 

73 

74 content = get_package_content( 

75 descr, local_files_only=keep_remote_files_as_references 

76 ) 

77 

78 metadata = content[BIOIMAGEIO_YAML] 

79 assert isinstance(metadata, dict) 

80 manifest = dict(metadata) 

81 

82 # only admins can upload a resource with a version 

83 artifact_version = "stage" # if descr.version is None else str(descr.version) 

84 

85 # Create new model 

86 r = httpx.post( 

87 settings.hypha_upload, 

88 json={ 

89 "parent_id": "bioimage-io/bioimage.io", 

90 "alias": ( 

91 descr.id or "{animal_adjective}-{animal}" 

92 ), # TODO: adapt for non-model uploads, 

93 "type": descr.type, 

94 "manifest": manifest, 

95 "version": artifact_version, 

96 }, 

97 headers=( 

98 headers := { 

99 "Authorization": f"Bearer {settings.hypha_upload_token}", 

100 "Content-Type": "application/json", 

101 } 

102 ), 

103 ) 

104 

105 response = r.json() 

106 artifact_id = response.get("id") 

107 if artifact_id is None: 

108 try: 

109 logger.error("Response detail: {}", "".join(response["detail"])) 

110 except Exception: 

111 logger.error("Response: {}", response) 

112 

113 raise RuntimeError(f"Upload did not return resource id: {response}") 

114 else: 

115 logger.info("Uploaded resource description {}", artifact_id) 

116 

117 for file_name, file_source in content.items(): 

118 # Get upload URL for a file 

119 response = httpx.post( 

120 settings.hypha_upload.replace("/create", "/put_file"), 

121 json={ 

122 "artifact_id": artifact_id, 

123 "file_path": file_name, 

124 }, 

125 headers=headers, 

126 follow_redirects=True, 

127 ) 

128 upload_url = response.raise_for_status().json() 

129 

130 # Upload file to the provided URL 

131 if isinstance(file_source, collections.abc.Mapping): 

132 buf = io.BytesIO() 

133 write_yaml(file_source, buf) 

134 _ = buf.seek(0) 

135 files = {file_name: buf} 

136 else: 

137 files = {file_name: get_reader(file_source)} 

138 

139 response = httpx.put( 

140 upload_url, 

141 files=files, # pyright: ignore[reportArgumentType] 

142 # TODO: follow up on https://github.com/encode/httpx/discussions/3611 

143 headers={"Content-Type": ""}, # Important for S3 uploads 

144 follow_redirects=True, 

145 ) 

146 logger.info("Uploaded '{}' successfully", file_name) 

147 

148 # Update model status 

149 manifest["status"] = "request-review" 

150 response = httpx.post( 

151 settings.hypha_upload.replace("/create", "/edit"), 

152 json={ 

153 "artifact_id": artifact_id, 

154 "version": artifact_version, 

155 "manifest": manifest, 

156 }, 

157 headers=headers, 

158 follow_redirects=True, 

159 ) 

160 logger.info( 

161 "Updated status of {}/{} to 'request-review'", artifact_id, artifact_version 

162 ) 

163 logger.warning( 

164 "Upload successfull. Please note that the uploaded resource might not be available for download immediately." 

165 ) 

166 with get_validation_context().replace(perform_io_checks=False): 

167 return HttpUrl( 

168 f"https://hypha.aicell.io/bioimage-io/artifacts/{artifact_id}/files/rdf.yaml?version={artifact_version}" 

169 )