Skip to content

bossdb.py

activate(schema_name, *, create_schema=True, create_tables=True, linking_module=None)

Activate this schema

Parameters:

Name Type Description Default
schema_name str

schema name on the database server to activate the bossdb schema

 required
create_schema bool

when True (default), create schema in the database if it does not yet exist.

 True
create_tables bool

when True (default), create schema tables in the database if they do not yet exist.

 True
linking_module str

A string containing the module name or module containing the required dependencies to activate the schema.

 None

Dependencies:

Tables

volume.VolumeSegmentation: A parent table to VolumeUploadTask volume.VoxelSize: A dependency of VolumeUpload

Functions

get_volume_root_data_dir: Returns absolute path for root data director(y/ies) with all volumetric data, as a list of string(s).

Source code in element_zstack/bossdb.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def activate(
    schema_name: str,
    *,
    create_schema: bool = True,
    create_tables: bool = True,
    linking_module: str = None,
):
    """Activate this schema

    Args:
        schema_name (str): schema name on the database server to activate the `bossdb` schema
        create_schema (bool): when True (default), create schema in the database if it
                            does not yet exist.
        create_tables (bool): when True (default), create schema tables in the database
                             if they do not yet exist.
        linking_module (str): A string containing the module name or module containing
            the required dependencies to activate the schema.

    Dependencies:
    Tables:
        volume.VolumeSegmentation: A parent table to VolumeUploadTask
        volume.VoxelSize: A dependency of VolumeUpload
    Functions:
        get_volume_root_data_dir: Returns absolute path for root data
        director(y/ies) with all volumetric data, as a list of string(s).
    """

    if isinstance(linking_module, str):
        linking_module = importlib.import_module(linking_module)
    assert inspect.ismodule(
        linking_module
    ), "The argument 'linking_module' must be a module's name or a module"

    global _linking_module
    _linking_module = linking_module

    schema.activate(
        schema_name,
        create_schema=create_schema,
        create_tables=create_tables,
        add_objects=_linking_module.__dict__,
    )

get_volume_root_data_dir()

Fetches absolute data path to volume data directories.

The absolute path here is used as a reference for all downstream relative paths used in DataJoint.

Returns:

Type Description
list

A list of the absolute path(s) to volume data directories.
Source code in element_zstack/bossdb.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
def get_volume_root_data_dir() -> list:
    """Fetches absolute data path to volume data directories.

    The absolute path here is used as a reference for all downstream relative paths used in DataJoint.

    Returns:
        A list of the absolute path(s) to volume data directories.
    """
    root_directories = _linking_module.get_volume_root_data_dir()
    if isinstance(root_directories, (str, Path)):
        root_directories = [root_directories]

    return root_directories

VolumeUploadTask

Bases: dj.Manual

Define the image and segmentation data to upload to BossDB.

Attributes:

Name Type Description
volume.Segmentation foreign key

Primary key from volume.Segmentation.
collection_name str

Name of the collection on BossDB.
experiment_name str

Name of the experiment on BossDB.
channel_name str

Name of the channel on BossDB.
Source code in element_zstack/bossdb.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@schema
class VolumeUploadTask(dj.Manual):
    """Define the image and segmentation data to upload to BossDB.

    Attributes:
        volume.Segmentation (foreign key): Primary key from `volume.Segmentation`.
        collection_name (str): Name of the collection on BossDB.
        experiment_name (str): Name of the experiment on BossDB.
        channel_name (str): Name of the channel on BossDB.
    """

    definition = """
    -> volume.Segmentation
    ---
    collection_name: varchar(64)
    experiment_name: varchar(64)
    channel_name: varchar(64)
    """

VolumeUpload

Bases: dj.Computed

Upload image and segmentation data to BossDB, and store the BossDB and Neuroglancer URLs.

Attributes:

Name Type Description
VolumeUploadTask foreign key

Primary key from VolumeUploadTask.
volume.VoxelSize foreign key

Primary key from volume.VoxelSize.
Source code in element_zstack/bossdb.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
@schema
class VolumeUpload(dj.Computed):
    """Upload image and segmentation data to BossDB, and store the BossDB and Neuroglancer URLs.

    Attributes:
        VolumeUploadTask (foreign key): Primary key from `VolumeUploadTask`.
        volume.VoxelSize (foreign key): Primary key from `volume.VoxelSize`.
    """

    definition = """
    -> VolumeUploadTask
    ---
    -> volume.VoxelSize
    """

    class WebAddress(dj.Part):
        """
        Attributes:
            VolumeUpload (foreign key): Primary key from `VolumeUpload`.
            upload_type (enum): 'image' (volumetric image), 'annotation' (segmentation data), or 'image+annotation' (segmentation overlayed on image).
            web_address_type (enum): 'bossdb' or 'neuroglancer'.
            web_address (str): URL for the data or visualization website.
        """

        definition = """
        -> master
        upload_type='image': enum('image', 'annotation', 'image+annotation')
        web_address_type='bossdb': enum('bossdb', 'neuroglancer')
        ---
        web_address: varchar(2048)
        """

    def get_neuroglancer_url(self, upload_type, collection, experiment, channel):
        base_url = f"boss://https://api.bossdb.io/{collection}/{experiment}/{channel}"
        if upload_type == "image":
            return "https://neuroglancer.bossdb.io/#!" + str(
                {
                    "layers": {
                        "type": "image",
                        "source": base_url,
                        "tab": "source",
                        "name": f"{experiment}",
                    }
                }
            )
        elif upload_type == "annotation":
            return "https://neuroglancer.bossdb.io/#!" + str(
                {
                    "layers": {
                        "type": "segmentation",
                        "source": base_url + "-seg",
                        "tab": "annotations",
                        "name": f"{channel}-seg",
                    }
                }
            )
        elif upload_type == "image+annotation":
            return "https://neuroglancer.bossdb.io/#!" + str(
                {
                    "layers": [
                        {
                            "type": "image",
                            "source": base_url,
                            "tab": "source",
                            "name": f"{experiment}",
                        },
                        {
                            "type": "segmentation",
                            "source": base_url + "-seg",
                            "tab": "annotations",
                            "name": f"{channel}-seg",
                        },
                    ]
                }
            )

    def make(self, key):
        """Upload data to bossdb."""

        collection, experiment, channel = (VolumeUploadTask & key).fetch1(
            "collection_name", "experiment_name", "channel_name"
        )

        voxel_width, voxel_height, voxel_depth = (volume.VoxelSize & key).fetch1(
            "width", "height", "depth"
        )

        description = ["image", "annotation", "image+annotation"]
        full_data = []
        boss_url = []
        neuroglancer_url = []

        volume_relative_path = (volume.Volume & key).fetch1("volume_file_path")
        volume_file_path = find_full_path(
            get_volume_root_data_dir(), volume_relative_path
        ).as_posix()
        volume_data = TiffFile(volume_file_path).asarray()

        full_data.append(volume_data)
        boss_url.append(f"bossdb://{collection}/{experiment}/{channel}")

        z_size, y_size, x_size = (volume.Volume & key).fetch1(
            "px_depth", "px_height", "px_width"
        )
        segmentation_data = np.zeros((z_size, y_size, x_size))

        mask_ids, x_mask_pix, y_mask_pix, z_mask_pix = (
            volume.Segmentation.Mask & key
        ).fetch("mask", "mask_xpix", "mask_ypix", "mask_zpix")

        for idx, mask in enumerate(mask_ids):
            segmentation_data[
                np.s_[z_mask_pix[idx], y_mask_pix[idx], x_mask_pix[idx]]
            ] = mask
        full_data.append(segmentation_data.astype("uint64"))

        boss_url.append(f"bossdb://{collection}/{experiment}/{channel}-seg")

        for url, data, desc in zip(boss_url, full_data, description[:2]):
            BossDBUpload(
                url=url,
                volume_data=data,
                data_description=desc,
                voxel_size=(voxel_depth, voxel_height, voxel_width),
                voxel_units="millimeters",
            ).upload()

        self.insert1(key)
        self.WebAddress.insert(
            [
                dict(
                    key,
                    upload_type=desc,
                    web_address_type="bossdb",
                    web_address=db_url,
                )
                for desc, db_url in list(zip(description[:2], boss_url))
            ]
        )

        for desc in description:
            neuroglancer_url.append(
                self.get_neuroglancer_url(desc, collection, experiment, channel)
            )
        self.WebAddress.insert(
            [
                dict(
                    key,
                    upload_type=desc,
                    web_address_type="neuroglancer",
                    web_address=url,
                )
                for desc, url in list(zip(description, neuroglancer_url))
            ]
        )

WebAddress

Bases: dj.Part

Attributes:

Name Type Description
VolumeUpload foreign key

Primary key from VolumeUpload.
upload_type enum

'image' (volumetric image), 'annotation' (segmentation data), or 'image+annotation' (segmentation overlayed on image).
web_address_type enum

'bossdb' or 'neuroglancer'.
web_address str

URL for the data or visualization website.
Source code in element_zstack/bossdb.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
class WebAddress(dj.Part):
    """
    Attributes:
        VolumeUpload (foreign key): Primary key from `VolumeUpload`.
        upload_type (enum): 'image' (volumetric image), 'annotation' (segmentation data), or 'image+annotation' (segmentation overlayed on image).
        web_address_type (enum): 'bossdb' or 'neuroglancer'.
        web_address (str): URL for the data or visualization website.
    """

    definition = """
    -> master
    upload_type='image': enum('image', 'annotation', 'image+annotation')
    web_address_type='bossdb': enum('bossdb', 'neuroglancer')
    ---
    web_address: varchar(2048)
    """

make(key)

Upload data to bossdb.

Source code in element_zstack/bossdb.py 
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def make(self, key):
    """Upload data to bossdb."""

    collection, experiment, channel = (VolumeUploadTask & key).fetch1(
        "collection_name", "experiment_name", "channel_name"
    )

    voxel_width, voxel_height, voxel_depth = (volume.VoxelSize & key).fetch1(
        "width", "height", "depth"
    )

    description = ["image", "annotation", "image+annotation"]
    full_data = []
    boss_url = []
    neuroglancer_url = []

    volume_relative_path = (volume.Volume & key).fetch1("volume_file_path")
    volume_file_path = find_full_path(
        get_volume_root_data_dir(), volume_relative_path
    ).as_posix()
    volume_data = TiffFile(volume_file_path).asarray()

    full_data.append(volume_data)
    boss_url.append(f"bossdb://{collection}/{experiment}/{channel}")

    z_size, y_size, x_size = (volume.Volume & key).fetch1(
        "px_depth", "px_height", "px_width"
    )
    segmentation_data = np.zeros((z_size, y_size, x_size))

    mask_ids, x_mask_pix, y_mask_pix, z_mask_pix = (
        volume.Segmentation.Mask & key
    ).fetch("mask", "mask_xpix", "mask_ypix", "mask_zpix")

    for idx, mask in enumerate(mask_ids):
        segmentation_data[
            np.s_[z_mask_pix[idx], y_mask_pix[idx], x_mask_pix[idx]]
        ] = mask
    full_data.append(segmentation_data.astype("uint64"))

    boss_url.append(f"bossdb://{collection}/{experiment}/{channel}-seg")

    for url, data, desc in zip(boss_url, full_data, description[:2]):
        BossDBUpload(
            url=url,
            volume_data=data,
            data_description=desc,
            voxel_size=(voxel_depth, voxel_height, voxel_width),
            voxel_units="millimeters",
        ).upload()

    self.insert1(key)
    self.WebAddress.insert(
        [
            dict(
                key,
                upload_type=desc,
                web_address_type="bossdb",
                web_address=db_url,
            )
            for desc, db_url in list(zip(description[:2], boss_url))
        ]
    )

    for desc in description:
        neuroglancer_url.append(
            self.get_neuroglancer_url(desc, collection, experiment, channel)
        )
    self.WebAddress.insert(
        [
            dict(
                key,
                upload_type=desc,
                web_address_type="neuroglancer",
                web_address=url,
            )
            for desc, url in list(zip(description, neuroglancer_url))
        ]
    )