| 1 | ### script to download opacities from DACE |
|---|
| 2 | ### adapted from https://github.com/bmorris3/shone/tree/refs/heads/opacity-downloads |
|---|
| 3 | |
|---|
| 4 | import logging |
|---|
| 5 | import warnings |
|---|
| 6 | from functools import cached_property |
|---|
| 7 | import os |
|---|
| 8 | import tarfile |
|---|
| 9 | import shutil |
|---|
| 10 | |
|---|
| 11 | import numpy as np |
|---|
| 12 | import xarray as xr |
|---|
| 13 | from astropy.table import Table |
|---|
| 14 | from dace_query.opacity import Molecule, Atom |
|---|
| 15 | |
|---|
| 16 | from chemistry import species_name_to_common_isotopologue_name |
|---|
| 17 | |
|---|
| 18 | interp_kwargs = dict( |
|---|
| 19 | method='nearest', |
|---|
| 20 | kwargs=dict(fill_value="extrapolate") |
|---|
| 21 | ) |
|---|
| 22 | |
|---|
| 23 | __all__ = [ |
|---|
| 24 | 'download_molecule', |
|---|
| 25 | 'download_atom' |
|---|
| 26 | ] |
|---|
| 27 | |
|---|
| 28 | class AvailableOpacities: |
|---|
| 29 | @cached_property |
|---|
| 30 | def atomic(self): |
|---|
| 31 | return get_atomic_database() |
|---|
| 32 | |
|---|
| 33 | @cached_property |
|---|
| 34 | def molecular(self): |
|---|
| 35 | return get_molecular_database() |
|---|
| 36 | |
|---|
| 37 | def get_atomic_database_entry(self, atom, charge, line_list): |
|---|
| 38 | table = self.atomic |
|---|
| 39 | return table[( |
|---|
| 40 | (table['atom'] == atom) & |
|---|
| 41 | (table['line_list'] == line_list) & |
|---|
| 42 | (table['charge'] == charge) |
|---|
| 43 | )] |
|---|
| 44 | |
|---|
| 45 | def get_molecular_database_entry(self, isotopologue, line_list): |
|---|
| 46 | table = self.molecular |
|---|
| 47 | return table[( |
|---|
| 48 | (table['isotopologue'] == isotopologue) & |
|---|
| 49 | (table['line_list'] == line_list) |
|---|
| 50 | )] |
|---|
| 51 | |
|---|
| 52 | def get_molecular_line_lists(self, isotopologue): |
|---|
| 53 | table = self.molecular |
|---|
| 54 | return set(table[table['isotopologue'] == isotopologue]['line_list']) |
|---|
| 55 | |
|---|
| 56 | def get_atomic_line_lists(self, atom): |
|---|
| 57 | table = self.atomic |
|---|
| 58 | return set(table[table['atom'] == atom]['line_list']) |
|---|
| 59 | |
|---|
| 60 | def get_atomic_latest_version(self, atom, charge, line_list): |
|---|
| 61 | table = self.atomic |
|---|
| 62 | matches = table[( |
|---|
| 63 | (table['atom'] == atom) & |
|---|
| 64 | (table['line_list'] == line_list) & |
|---|
| 65 | (table['charge'] == charge) |
|---|
| 66 | )] |
|---|
| 67 | return max(set(matches['version'])) |
|---|
| 68 | |
|---|
| 69 | def get_molecular_latest_version(self, isotopologue, line_list): |
|---|
| 70 | table = self.molecular |
|---|
| 71 | matches = table[( |
|---|
| 72 | (table['isotopologue'] == isotopologue) & |
|---|
| 73 | (table['line_list'] == line_list) |
|---|
| 74 | )] |
|---|
| 75 | return max(set(matches['version'])) |
|---|
| 76 | |
|---|
| 77 | def get_atomic_pT_range(self, atom, charge, line_list): |
|---|
| 78 | table = self.get_atomic_database_entry(atom, charge, line_list) |
|---|
| 79 | temperature_range = ( |
|---|
| 80 | int(table['temp_min_k'][0]), |
|---|
| 81 | int(table['temp_max_k'][0]) |
|---|
| 82 | ) |
|---|
| 83 | pressure_range = ( |
|---|
| 84 | float(table['pressure_min_exponent_b'][0]), |
|---|
| 85 | float(table['pressure_max_exponent_b'][0]) |
|---|
| 86 | ) |
|---|
| 87 | return temperature_range, pressure_range |
|---|
| 88 | |
|---|
| 89 | def get_molecular_pT_range(self, isotopologue, line_list): |
|---|
| 90 | table = self.get_molecular_database_entry(isotopologue, line_list) |
|---|
| 91 | temperature_range = ( |
|---|
| 92 | int(table['temp_min_k'][0]), |
|---|
| 93 | int(table['temp_max_k'][0]) |
|---|
| 94 | ) |
|---|
| 95 | pressure_range = ( |
|---|
| 96 | float(table['pressure_min_exponent_b'][0]), |
|---|
| 97 | float(table['pressure_max_exponent_b'][0]) |
|---|
| 98 | ) |
|---|
| 99 | return temperature_range, pressure_range |
|---|
| 100 | |
|---|
| 101 | |
|---|
| 102 | available_opacities = AvailableOpacities() |
|---|
| 103 | |
|---|
| 104 | |
|---|
| 105 | def get_atomic_database(): |
|---|
| 106 | db = Atom.query_database() |
|---|
| 107 | table = Table(db) |
|---|
| 108 | table.add_index('atom') |
|---|
| 109 | return table |
|---|
| 110 | |
|---|
| 111 | |
|---|
| 112 | def get_molecular_database(): |
|---|
| 113 | db = Molecule.query_database() |
|---|
| 114 | table = Table(db) |
|---|
| 115 | table.add_index('isotopologue') |
|---|
| 116 | return table |
|---|
| 117 | |
|---|
| 118 | class DownloadDace: |
|---|
| 119 | def __init__(self, |
|---|
| 120 | name='48Ti-16O', |
|---|
| 121 | line_list='Toto', |
|---|
| 122 | temperature_range=None, |
|---|
| 123 | pressure_range=None, |
|---|
| 124 | version=1, |
|---|
| 125 | output_dir="tmp", |
|---|
| 126 | force_download = False, |
|---|
| 127 | ): |
|---|
| 128 | self.output_dir = output_dir |
|---|
| 129 | self.name = name |
|---|
| 130 | self.line_list = line_list |
|---|
| 131 | self.temperature_range = temperature_range |
|---|
| 132 | self.pressure_range = pressure_range |
|---|
| 133 | self.version = version |
|---|
| 134 | self.force_download = force_download |
|---|
| 135 | |
|---|
| 136 | def dace_download_molecule(self): |
|---|
| 137 | os.makedirs(self.output_dir, exist_ok=True) |
|---|
| 138 | archive_name = self.name + '__' + self.line_list + '.tar.gz' |
|---|
| 139 | path = os.path.join(self.output_dir, archive_name) |
|---|
| 140 | if not self.force_download and os.path.exists(path): |
|---|
| 141 | print("Molecule already downloaded") |
|---|
| 142 | return path |
|---|
| 143 | Molecule.download( |
|---|
| 144 | self.name, |
|---|
| 145 | self.line_list, |
|---|
| 146 | float(self.version), |
|---|
| 147 | self.temperature_range, |
|---|
| 148 | self.pressure_range, |
|---|
| 149 | output_directory=self.output_dir, |
|---|
| 150 | output_filename=archive_name |
|---|
| 151 | ) |
|---|
| 152 | |
|---|
| 153 | return path |
|---|
| 154 | |
|---|
| 155 | |
|---|
| 156 | def dace_download_atom(self, charge=0): |
|---|
| 157 | os.makedirs(self.output_dir, exist_ok=True) |
|---|
| 158 | archive_name = self.name + '__' + self.line_list + '.tar.gz' |
|---|
| 159 | Atom.download( |
|---|
| 160 | self.name, charge, |
|---|
| 161 | self.line_list, float(self.version), |
|---|
| 162 | self.temperature_range, |
|---|
| 163 | self.pressure_range, |
|---|
| 164 | output_directory=self.output_dir, |
|---|
| 165 | output_filename=archive_name |
|---|
| 166 | ) |
|---|
| 167 | #### Atoms don't download anything for now, reason unknown |
|---|
| 168 | |
|---|
| 169 | return os.path.join(self.output_dir, archive_name) |
|---|
| 170 | |
|---|
| 171 | |
|---|
| 172 | def untar_bin_files(self, archive_name): |
|---|
| 173 | def bin_files(members): |
|---|
| 174 | for tarinfo in members: |
|---|
| 175 | if os.path.splitext(tarinfo.name)[1] == ".bin": |
|---|
| 176 | yield tarinfo |
|---|
| 177 | |
|---|
| 178 | path = os.path.join(self.output_dir, self.name + '__' + self.line_list) |
|---|
| 179 | with tarfile.open(archive_name, 'r') as tar: |
|---|
| 180 | tar.extractall(path=path , members=bin_files(tar)) |
|---|
| 181 | |
|---|
| 182 | return path |
|---|
| 183 | |
|---|
| 184 | |
|---|
| 185 | def opacity_dir_to_netcdf(self, opacity_dir, outpath): |
|---|
| 186 | outpath = self.output_dir + '/' + outpath |
|---|
| 187 | if os.path.exists(outpath): |
|---|
| 188 | print(f"{outpath} already exists") |
|---|
| 189 | return |
|---|
| 190 | temperature_grid = [] |
|---|
| 191 | pressure_grid = [] |
|---|
| 192 | wl_end = np.inf |
|---|
| 193 | |
|---|
| 194 | for dirpath, dirnames, filenames in os.walk(opacity_dir): |
|---|
| 195 | for filename in filenames: |
|---|
| 196 | if not filename.endswith('.bin'): |
|---|
| 197 | continue |
|---|
| 198 | |
|---|
| 199 | # Wavenumber points from range given in the file names |
|---|
| 200 | temperature = int(filename.split('_')[3]) |
|---|
| 201 | sign = 1 if filename.split('_')[4][0] == 'p' else -1 |
|---|
| 202 | pressure = 10 ** (sign * float(filename.split('_')[4][1:].split('.')[0]) / 100) |
|---|
| 203 | |
|---|
| 204 | wl_start = int(filename.split('_')[1]) |
|---|
| 205 | wl_end = min(wl_end, int(filename.split('_')[2])) |
|---|
| 206 | wlen = np.arange(wl_start, wl_end, 0.01) |
|---|
| 207 | |
|---|
| 208 | # catch divide by zero warning here: |
|---|
| 209 | with warnings.catch_warnings(): |
|---|
| 210 | warnings.simplefilter('ignore', RuntimeWarning) |
|---|
| 211 | |
|---|
| 212 | # Convert to micron |
|---|
| 213 | wavelength = 1 / wlen / 1e-4 |
|---|
| 214 | |
|---|
| 215 | unique_wavelengths = wavelength[1:][::-1] |
|---|
| 216 | temperature_grid.append(temperature) |
|---|
| 217 | pressure_grid.append(pressure) |
|---|
| 218 | |
|---|
| 219 | tgrid = np.sort(list(set(temperature_grid))) |
|---|
| 220 | pgrid = np.sort(list(set(pressure_grid))) |
|---|
| 221 | |
|---|
| 222 | if len(pgrid) == 1: |
|---|
| 223 | extrapolate_pgrid = True |
|---|
| 224 | pgrid = np.concatenate([pgrid, 10 ** (-1 * np.log10(pgrid))]) |
|---|
| 225 | else: |
|---|
| 226 | extrapolate_pgrid = False |
|---|
| 227 | |
|---|
| 228 | opacity_grid = np.zeros( |
|---|
| 229 | (len(tgrid), len(pgrid), len(unique_wavelengths)), dtype='float32' |
|---|
| 230 | ) |
|---|
| 231 | |
|---|
| 232 | for dirpath, dirnames, filenames in os.walk(opacity_dir): |
|---|
| 233 | for filename in filenames: |
|---|
| 234 | if not filename.endswith('.bin'): |
|---|
| 235 | continue |
|---|
| 236 | |
|---|
| 237 | opacity = np.fromfile( |
|---|
| 238 | os.path.join(dirpath, filename), dtype=np.float32 |
|---|
| 239 | )[1:][::-1] |
|---|
| 240 | |
|---|
| 241 | # Wavenumber points from range given in the file names |
|---|
| 242 | temperature = int(filename.split('_')[3]) |
|---|
| 243 | sign = 1 if filename.split('_')[4][0] == 'p' else -1 |
|---|
| 244 | pressure = 10 ** (sign * float(filename.split('_')[4][1:].split('.')[0]) / 100) |
|---|
| 245 | |
|---|
| 246 | temperature_ind = np.argmin(np.abs(tgrid - temperature)) |
|---|
| 247 | pressure_ind = np.argmin(np.abs(pgrid - pressure)) |
|---|
| 248 | |
|---|
| 249 | opacity_grid[temperature_ind, pressure_ind, :] = opacity[:len(unique_wavelengths)] |
|---|
| 250 | |
|---|
| 251 | if extrapolate_pgrid: |
|---|
| 252 | for dirpath, dirnames, filenames in os.walk(opacity_dir): |
|---|
| 253 | for filename in filenames: |
|---|
| 254 | opacity = np.fromfile( |
|---|
| 255 | os.path.join(dirpath, filename), dtype=np.float32 |
|---|
| 256 | )[1:][::-1] |
|---|
| 257 | |
|---|
| 258 | # Wavenumber points from range given in the file names |
|---|
| 259 | temperature = int(filename.split('_')[3]) |
|---|
| 260 | # *Flip the sign for the extrapolated grid point in pressure* |
|---|
| 261 | sign = -1 if filename.split('_')[4][0] == 'p' else 1 |
|---|
| 262 | pressure = 10 ** (sign * float(filename.split('_')[4][1:].split('.')[0]) / 100) |
|---|
| 263 | |
|---|
| 264 | temperature_ind = np.argmin(np.abs(tgrid - temperature)) |
|---|
| 265 | pressure_ind = np.argmin(np.abs(pgrid - pressure)) |
|---|
| 266 | |
|---|
| 267 | opacity_grid[temperature_ind, pressure_ind, :] = opacity |
|---|
| 268 | |
|---|
| 269 | ds = xr.Dataset( |
|---|
| 270 | data_vars=dict( |
|---|
| 271 | opacity=(["temperature", "pressure", "wavelength"], |
|---|
| 272 | opacity_grid) |
|---|
| 273 | ), |
|---|
| 274 | coords=dict( |
|---|
| 275 | temperature=(["temperature"], tgrid), |
|---|
| 276 | pressure=(["pressure"], pgrid), |
|---|
| 277 | wavelength=unique_wavelengths |
|---|
| 278 | ) |
|---|
| 279 | ) |
|---|
| 280 | if not os.path.exists(os.path.dirname(outpath)): |
|---|
| 281 | os.makedirs(os.path.dirname(outpath), exist_ok=True) |
|---|
| 282 | |
|---|
| 283 | ds.to_netcdf(outpath if outpath.endswith(".nc") else outpath + '.nc', |
|---|
| 284 | encoding={'opacity': {'dtype': 'float32'}}) |
|---|
| 285 | |
|---|
| 286 | |
|---|
| 287 | def clean_up(self, bin_dir, archive_name): |
|---|
| 288 | os.remove(archive_name) |
|---|
| 289 | shutil.rmtree(bin_dir) |
|---|
| 290 | |
|---|
| 291 | |
|---|
| 292 | def download_molecule( |
|---|
| 293 | molecule=None, |
|---|
| 294 | isotopologue=None, |
|---|
| 295 | line_list='first-found', |
|---|
| 296 | temperature_range=None, |
|---|
| 297 | pressure_range=None, |
|---|
| 298 | version=None, |
|---|
| 299 | output_dir="datadir/dace", |
|---|
| 300 | force_download=False, |
|---|
| 301 | ): |
|---|
| 302 | """ |
|---|
| 303 | Download molecular opacity data from DACE. |
|---|
| 304 | |
|---|
| 305 | .. warning:: |
|---|
| 306 | This generates *very* large files. Only run this |
|---|
| 307 | method if you have ~6 GB available per molecule. |
|---|
| 308 | |
|---|
| 309 | Parameters |
|---|
| 310 | ---------- |
|---|
| 311 | isotopologue : str |
|---|
| 312 | For example, "1H2-16O" for water. |
|---|
| 313 | molecule : str |
|---|
| 314 | Common name for the molecule, for example: "H2O" |
|---|
| 315 | line_list : str, default is ``'first-found'``, optional |
|---|
| 316 | For example, "POKAZATEL" for water. By default, the first available |
|---|
| 317 | line list for this isotopologue is chosen. |
|---|
| 318 | temperature_range : tuple, optional |
|---|
| 319 | Tuple of integers specifying the min and max |
|---|
| 320 | temperature requested. Defaults to the full |
|---|
| 321 | range of available temperatures. |
|---|
| 322 | pressure_range : tuple, optional |
|---|
| 323 | Tuple of floags specifying the log base 10 of the |
|---|
| 324 | min and max pressure [bar] requested. Defaults to the full |
|---|
| 325 | range of available pressures. |
|---|
| 326 | version : float, optional |
|---|
| 327 | Version number of the line list in DACE. Defaults to the |
|---|
| 328 | latest version. |
|---|
| 329 | """ |
|---|
| 330 | if molecule is not None: |
|---|
| 331 | isotopologue = species_name_to_common_isotopologue_name(molecule) |
|---|
| 332 | |
|---|
| 333 | available_line_lists = available_opacities.get_molecular_line_lists(isotopologue) |
|---|
| 334 | print(f"Available line lists for {isotopologue}:") |
|---|
| 335 | print(available_line_lists) |
|---|
| 336 | |
|---|
| 337 | if line_list is None: |
|---|
| 338 | return |
|---|
| 339 | |
|---|
| 340 | if line_list == 'first-found': |
|---|
| 341 | line_list = sorted(list(available_line_lists)).pop() |
|---|
| 342 | logging.warning(f"Using first-found line list for {isotopologue}: '{line_list}'") |
|---|
| 343 | |
|---|
| 344 | elif line_list not in available_line_lists: |
|---|
| 345 | raise ValueError(f"The requested '{line_list}' is not in the set of " |
|---|
| 346 | f"available line lists {available_line_lists}.") |
|---|
| 347 | |
|---|
| 348 | if version is None: |
|---|
| 349 | version = available_opacities.get_molecular_latest_version(isotopologue, line_list) |
|---|
| 350 | logging.warning(f"Using latest version of the line " |
|---|
| 351 | f"list '{line_list}' for {isotopologue}: {version}") |
|---|
| 352 | |
|---|
| 353 | if temperature_range is None or pressure_range is None: |
|---|
| 354 | dace_temp_range, dace_press_range = available_opacities.get_molecular_pT_range( |
|---|
| 355 | isotopologue, line_list |
|---|
| 356 | ) |
|---|
| 357 | |
|---|
| 358 | if temperature_range is None: |
|---|
| 359 | temperature_range = dace_temp_range |
|---|
| 360 | |
|---|
| 361 | if pressure_range is None: |
|---|
| 362 | pressure_range = dace_press_range |
|---|
| 363 | |
|---|
| 364 | download_dace = DownloadDace(isotopologue, line_list, temperature_range, pressure_range, version, output_dir, force_download=force_download) |
|---|
| 365 | archive_name = download_dace.dace_download_molecule() |
|---|
| 366 | |
|---|
| 367 | bin_dir = download_dace.untar_bin_files(archive_name) |
|---|
| 368 | |
|---|
| 369 | download_dace.opacity_dir_to_netcdf(bin_dir, isotopologue + '__' + line_list + '.nc') |
|---|
| 370 | # clean_up(bin_dir, archive_name) |
|---|
| 371 | return isotopologue + '__' + line_list + '.nc' |
|---|
| 372 | |
|---|
| 373 | |
|---|
| 374 | def download_atom(atom, charge, line_list='first-found', |
|---|
| 375 | temperature_range=None, pressure_range=None, version=None, output_dir="tmp"): |
|---|
| 376 | """ |
|---|
| 377 | Download atomic opacity data from DACE. |
|---|
| 378 | |
|---|
| 379 | .. warning:: |
|---|
| 380 | This generates *very* large files. Only run this |
|---|
| 381 | method if you have ~6 GB available per molecule. |
|---|
| 382 | |
|---|
| 383 | Parameters |
|---|
| 384 | ---------- |
|---|
| 385 | atom : str |
|---|
| 386 | For example, "Na" for sodium. |
|---|
| 387 | charge : int |
|---|
| 388 | For example, 0 for neutral. |
|---|
| 389 | line_list : str, default is ``'first-found'``, optional |
|---|
| 390 | For example, "Kurucz". By default, the first available |
|---|
| 391 | line list for this atom/charge is chosen. |
|---|
| 392 | temperature_range : tuple, optional |
|---|
| 393 | Tuple of integers specifying the min and max |
|---|
| 394 | temperature requested. Defaults to the full |
|---|
| 395 | range of available temperatures. |
|---|
| 396 | pressure_range : tuple, optional |
|---|
| 397 | Tuple of floags specifying the log base 10 of the |
|---|
| 398 | min and max pressure [bar] requested. Defaults to the full |
|---|
| 399 | range of available pressures. |
|---|
| 400 | version : float, optional |
|---|
| 401 | Version number of the line list in DACE. Defaults to the |
|---|
| 402 | latest version. |
|---|
| 403 | """ |
|---|
| 404 | available_line_lists = available_opacities.get_atomic_line_lists(atom) |
|---|
| 405 | print(f"Available line lists for {atom}:") |
|---|
| 406 | print(available_line_lists) |
|---|
| 407 | |
|---|
| 408 | if line_list is None: |
|---|
| 409 | return |
|---|
| 410 | |
|---|
| 411 | if line_list == 'first-found': |
|---|
| 412 | line_list = sorted(list(available_line_lists)).pop() |
|---|
| 413 | logging.warning(f"Using first-found line list for {atom}: '{line_list}'") |
|---|
| 414 | |
|---|
| 415 | elif line_list not in available_line_lists: |
|---|
| 416 | raise ValueError(f"The requested '{line_list}' is not in the set of " |
|---|
| 417 | f"available line lists {available_line_lists}.") |
|---|
| 418 | |
|---|
| 419 | if temperature_range is None or pressure_range is None: |
|---|
| 420 | dace_temp_range, dace_press_range = available_opacities.get_atomic_pT_range( |
|---|
| 421 | atom, charge, line_list |
|---|
| 422 | ) |
|---|
| 423 | |
|---|
| 424 | if version is None: |
|---|
| 425 | version = available_opacities.get_atomic_latest_version(atom, charge, line_list) |
|---|
| 426 | logging.warning(f"Using latest version of the line " |
|---|
| 427 | f"list '{line_list}' for {atom}: {version}") |
|---|
| 428 | |
|---|
| 429 | if temperature_range is None: |
|---|
| 430 | temperature_range = dace_temp_range |
|---|
| 431 | |
|---|
| 432 | if pressure_range is None: |
|---|
| 433 | pressure_range = dace_press_range |
|---|
| 434 | |
|---|
| 435 | download_dace = DownloadDace(atom, line_list, temperature_range, pressure_range, version, output_dir) |
|---|
| 436 | |
|---|
| 437 | archive_name = download_dace.dace_download_atom(charge) |
|---|
| 438 | bin_dir = download_dace.untar_bin_files(archive_name) |
|---|
| 439 | |
|---|
| 440 | download_dace.opacity_dir_to_netcdf(bin_dir, atom + '_' + str(int(charge)) + '__' + line_list + '.nc') |
|---|
| 441 | # download_dace.clean_up(bin_dir, archive_name) |
|---|
