Coverage for bioimageio/spec/_upload.py: 28%

1import collections.abc 

2import io 

3from typing import Union 

4from zipfile import ZipFile 

5 

6import httpx 

7from loguru import logger 

8 

9from bioimageio.spec._io import load_description 

10 

11from ._description import ( 

12 InvalidDescr, 

13 ResourceDescr, 

14 build_description, 

15) 

16from ._internal._settings import settings 

17from ._internal.common_nodes import ResourceDescrBase 

18from ._internal.io import BioimageioYamlContent, get_reader 

19from ._internal.io_basics import BIOIMAGEIO_YAML 

20from ._internal.io_utils import write_yaml 

21from ._internal.validation_context import get_validation_context 

22from ._package import get_resource_package_content 

23from .common import HttpUrl, PermissiveFileSource 

24 

25 

26# TODO: remove alpha stage warning 

27def upload( 

28 source: Union[PermissiveFileSource, ZipFile, ResourceDescr, BioimageioYamlContent], 

29 /, 

30) -> HttpUrl: 

31 """Upload a new resource description (version) to the hypha server to be shared at bioimage.io. 

32 To edit an existing resource **version**, please login to https://bioimage.io and use the web interface. 

33 

34 WARNING: This upload function is in alpha stage and might change in the future. 

35 

36 Args: 

37 source: The resource description to upload. 

38 

39 Returns: 

40 A URL to the uploaded resource description. 

41 Note: It might take some time until the resource is processed and available for download from the returned URL. 

42 """ 

43 

44 if settings.hypha_upload_token is None: 

45 raise ValueError( 

46 """ 

47Upload token is not set. Please set BIOIMAGEIO_HYPHA_UPLOAD_TOKEN in your environment variables. 

48By setting this token you agree to our terms of service at https://bioimage.io/#/toc. 

49 

50How to obtain a token: 

51 1. Login to https://bioimage.io 

52 2. Generate a new token at https://bioimage.io/#/api?tab=hypha-rpc 

53""" 

54 ) 

55 

56 if isinstance(source, ResourceDescrBase): 

57 # If source is already a ResourceDescr, we can use it directly 

58 descr = source 

59 elif isinstance(source, dict): 

60 descr = build_description(source) 

61 else: 

62 descr = load_description(source) 

63 

64 if isinstance(descr, InvalidDescr): 

65 raise ValueError("Uploading invalid resource descriptions is not allowed.") 

66 

67 if descr.type != "model": 

68 raise NotImplementedError( 

69 f"For now, only model resources can be uploaded (got type={descr.type})." 

70 ) 

71 

72 if descr.id is not None: 

73 raise ValueError( 

74 "You cannot upload a resource with an id. Please remove the id from the description and make sure to upload a new non-existing resource. To edit an existing resource, please use the web interface at https://bioimage.io." 

75 ) 

76 

77 content = get_resource_package_content(descr) 

78 

79 metadata = content[BIOIMAGEIO_YAML] 

80 assert isinstance(metadata, dict) 

81 manifest = dict(metadata) 

82 

83 # only admins can upload a resource with a version 

84 artifact_version = "stage" # if descr.version is None else str(descr.version) 

85 

86 # Create new model 

87 r = httpx.post( 

88 settings.hypha_upload, 

89 json={ 

90 "parent_id": "bioimage-io/bioimage.io", 

91 "alias": ( 

92 descr.id or "{animal_adjective}-{animal}" 

93 ), # TODO: adapt for non-model uploads, 

94 "type": descr.type, 

95 "manifest": manifest, 

96 "version": artifact_version, 

97 }, 

98 headers=( 

99 headers := { 

100 "Authorization": f"Bearer {settings.hypha_upload_token}", 

101 "Content-Type": "application/json", 

102 } 

103 ), 

104 ) 

105 

106 response = r.json() 

107 artifact_id = response.get("id") 

108 if artifact_id is None: 

109 try: 

110 logger.error("Response detail: {}", "".join(response["detail"])) 

111 except Exception: 

112 logger.error("Response: {}", response) 

113 

114 raise RuntimeError(f"Upload did not return resource id: {response}") 

115 else: 

116 logger.info("Uploaded resource description {}", artifact_id) 

117 

118 for file_name, file_source in content.items(): 

119 # Get upload URL for a file 

120 response = httpx.post( 

121 settings.hypha_upload.replace("/create", "/put_file"), 

122 json={ 

123 "artifact_id": artifact_id, 

124 "file_path": file_name, 

125 }, 

126 headers=headers, 

127 follow_redirects=True, 

128 ) 

129 upload_url = response.raise_for_status().json() 

130 

131 # Upload file to the provided URL 

132 if isinstance(file_source, collections.abc.Mapping): 

133 buf = io.BytesIO() 

134 write_yaml(file_source, buf) 

135 files = {file_name: buf} 

136 else: 

137 files = {file_name: get_reader(file_source)} 

138 

139 response = httpx.put( 

140 upload_url, 

141 files=files, # pyright: ignore[reportArgumentType] 

142 # TODO: follow up on https://github.com/encode/httpx/discussions/3611 

143 headers={"Content-Type": ""}, # Important for S3 uploads 

144 follow_redirects=True, 

145 ) 

146 logger.info("Uploaded '{}' successfully", file_name) 

147 

148 # Update model status 

149 manifest["status"] = "request-review" 

150 response = httpx.post( 

151 settings.hypha_upload.replace("/create", "/edit"), 

152 json={ 

153 "artifact_id": artifact_id, 

154 "version": artifact_version, 

155 "manifest": manifest, 

156 }, 

157 headers=headers, 

158 follow_redirects=True, 

159 ) 

160 logger.info( 

161 "Updated status of {}/{} to 'request-review'", artifact_id, artifact_version 

162 ) 

163 logger.warning( 

164 "Upload successfull. Please note that the uploaded resource might not be available for download immediately." 

165 ) 

166 with get_validation_context().replace(perform_io_checks=False): 

167 return HttpUrl( 

168 f"https://hypha.aicell.io/bioimage-io/artifacts/{artifact_id}/files/rdf.yaml?version={artifact_version}" 

169 )