Coverage for src/bioimageio/spec/_upload.py: 81%

1from __future__ import annotations 

2 

3import collections.abc 

4import io 

5from zipfile import ZipFile 

6 

7import httpx 

8from loguru import logger 

9 

10from ._description import InvalidDescr, ResourceDescr, build_description 

11from ._internal._settings import settings 

12from ._internal.common_nodes import ResourceDescrBase 

13from ._internal.io import BioimageioYamlContent, get_reader 

14from ._internal.io_basics import BIOIMAGEIO_YAML 

15from ._internal.io_utils import write_yaml 

16from ._internal.validation_context import get_validation_context 

17from ._io import load_description 

18from ._package import get_package_content 

19from .common import HttpUrl, PermissiveFileSource 

20 

21 

22# TODO: remove alpha stage warning 

23def upload( 

24 source: PermissiveFileSource | ZipFile | ResourceDescr | BioimageioYamlContent, 

25 /, 

26 keep_remote_files_as_references: bool = False, 

27) -> HttpUrl: 

28 """Upload a new resource description (version) to the hypha server to be shared at bioimage.io. 

29 To edit an existing resource **version**, please login to https://bioimage.io and use the web interface. 

30 

31 WARNING: This upload function is in alpha stage and might change in the future. 

32 

33 Args: 

34 source: The resource description to upload. 

35 keep_remote_files_as_references: If True, remote files will be kept as references and not downloaded and uploaded to the server. 

36 

37 Returns: 

38 A URL to the uploaded resource description. 

39 Note: It might take some time until the resource is processed and available for download from the returned URL. 

40 """ 

41 

42 if settings.hypha_upload_token is None: 

43 raise ValueError( 

44 """ 

45Upload token is not set. Please set BIOIMAGEIO_HYPHA_UPLOAD_TOKEN in your environment variables. 

46By setting this token you agree to our terms of service at https://bioimage.io/#/toc. 

47 

48How to obtain a token: 

49 1. Login to https://bioimage.io 

50 2. Generate a new token at https://bioimage.io/#/api?tab=hypha-rpc 

51""" 

52 ) 

53 

54 if isinstance(source, ResourceDescrBase): 

55 # If source is already a ResourceDescr, we can use it directly 

56 descr = source 

57 elif isinstance(source, dict): 

58 descr = build_description(source) 

59 else: 

60 descr = load_description(source) 

61 

62 if isinstance(descr, InvalidDescr): 

63 raise ValueError("Uploading invalid resource descriptions is not allowed.") 

64 

65 if descr.type != "model": 

66 raise NotImplementedError( 

67 f"For now, only model resources can be uploaded (got type={descr.type})." 

68 ) 

69 

70 if descr.id is not None: 

71 raise ValueError( 

72 "You cannot upload a resource with an id. Please remove the id from the description and make sure to upload a new non-existing resource. To edit an existing resource, please use the web interface at https://bioimage.io." 

73 ) 

74 

75 content = get_package_content( 

76 descr, local_files_only=keep_remote_files_as_references 

77 ) 

78 

79 metadata = content[BIOIMAGEIO_YAML] 

80 assert isinstance(metadata, dict) 

81 manifest = dict(metadata) 

82 

83 # only admins can upload a resource with a version 

84 artifact_version = "stage" # if descr.version is None else str(descr.version) 

85 

86 # Create new model 

87 r = httpx.post( 

88 settings.hypha_upload, 

89 json={ 

90 "parent_id": "bioimage-io/bioimage.io", 

91 "alias": ( 

92 descr.id or "{animal_adjective}-{animal}" 

93 ), # TODO: adapt for non-model uploads, 

94 "type": descr.type, 

95 "manifest": manifest, 

96 "version": artifact_version, 

97 }, 

98 headers=( 

99 headers := { 

100 "Authorization": f"Bearer {settings.hypha_upload_token}", 

101 "Content-Type": "application/json", 

102 } 

103 ), 

104 ) 

105 

106 response = r.json() 

107 artifact_id = response.get("id") 

108 if artifact_id is None: 

109 try: 

110 logger.error("Response detail: {}", "".join(response["detail"])) 

111 except Exception: 

112 logger.error("Response: {}", response) 

113 

114 raise RuntimeError(f"Upload did not return resource id: {response}") 

115 else: 

116 logger.info("Uploaded resource description {}", artifact_id) 

117 

118 for file_name, file_source in content.items(): 

119 # Get upload URL for a file 

120 response = httpx.post( 

121 settings.hypha_upload.replace("/create", "/put_file"), 

122 json={ 

123 "artifact_id": artifact_id, 

124 "file_path": file_name, 

125 }, 

126 headers=headers, 

127 follow_redirects=True, 

128 ) 

129 upload_url = response.raise_for_status().json() 

130 

131 # Upload file to the provided URL 

132 if isinstance(file_source, collections.abc.Mapping): 

133 buf = io.BytesIO() 

134 write_yaml(file_source, buf) 

135 _ = buf.seek(0) 

136 files = {file_name: buf} 

137 else: 

138 files = {file_name: get_reader(file_source)} 

139 

140 response = httpx.put( 

141 upload_url, 

142 files=files, # pyright: ignore[reportArgumentType] 

143 # TODO: follow up on https://github.com/encode/httpx/discussions/3611 

144 headers={"Content-Type": ""}, # Important for S3 uploads 

145 follow_redirects=True, 

146 ) 

147 logger.info("Uploaded '{}' successfully", file_name) 

148 

149 # Update model status 

150 manifest["status"] = "request-review" 

151 response = httpx.post( 

152 settings.hypha_upload.replace("/create", "/edit"), 

153 json={ 

154 "artifact_id": artifact_id, 

155 "version": artifact_version, 

156 "manifest": manifest, 

157 }, 

158 headers=headers, 

159 follow_redirects=True, 

160 ) 

161 logger.info( 

162 "Updated status of {}/{} to 'request-review'", artifact_id, artifact_version 

163 ) 

164 logger.warning( 

165 "Upload successfull. Please note that the uploaded resource might not be available for download immediately." 

166 ) 

167 with get_validation_context().replace(perform_io_checks=False): 

168 return HttpUrl( 

169 f"https://hypha.aicell.io/bioimage-io/artifacts/{artifact_id}/files/rdf.yaml?version={artifact_version}" 

170 )