import random
import numpy as np
import matplotlib.pyplot as plt
import sys
sys.path.append("..")
from solarbench import FormatMLDataset, load_datasets, get_dataloader, DatasetsInfo
import solarbench.evaluation as evaluation
from demo_helpers import display_sample, display_tensor_img
Dataset meta information retrieval¶
The DatasetsInfo class allows you to retrieve the meta information for the datasets that you are interested in. You can query the information either by dataset names (DatasetsInfo.by_name) or by dataset features (GetDatasetsInfo.by_features).
Obtaining metadata for all datasets¶
DatasetsInfo.datasets allows you to retrieve a pandas DataFrame containing metadata for every dataset in SolarBench.
DatasetsInfo.datasets
DatasetsInfo.benchmarks allows you to retrieve a pandas Dataframe containing information about the available benchmark datasets composed of multiple SolarBench datasets.
DatasetsInfo.benchmarks
Query by dataset names¶
DatasetsInfo.by_name allows you to retrieve the information for a list of dataset names.
# Retrieve the meta information of specific datasets
dataset_list = ['SRRL-BMS', 'SKIPPD']
DatasetsInfo.by_name(dataset_list)
Query by dataset features¶
DatasetsInfo.by_features allows you to retrieve the information by dataset features, e.g., the location of the data, the data types contained in the dataset, etc. A complete list of available query parameters is available in our full documentation.
# Retrieve information for datasets that contain PV generation data
dataset_features = {'data_types':['pv']}
DatasetsInfo.by_features(dataset_features)
# Retrieve information for datasets that are collectioned in certain regions or countries,e.g., {'Country':['France']}
dataset_features = {'Region':['NA'], 'Country': ['United States']} # you are able to input multiple regions in the list
DatasetsInfo.by_features(dataset_features)
# Retrieve information for datasets that contain PV generation data and are collectioned in certain regions
dataset_features = {'Data_types':['pv'],'Region':['NA', 'OC']}
DatasetsInfo.by_features(dataset_features)
Fetching available/benchmark dataset names¶
For convenience, you can use DatasetInfo.datasets_list to obtain a Python list containing the names of the currently available datasets in SolarBench, and DatasetInfo.benchmarks_list for benchmark dataset names.
print("Datasets:", DatasetsInfo.datasets_list)
print("Benchmarks:", DatasetsInfo.benchmarks_list)
Dataset downloading¶
You can download or retrieve one or more datasets using load_datasets. If you would like to potentially speed up your download, you can set the num_proc keyword argument.
Once datasets are downloaded once, they are cached locally in your system.
dataset_list = ['SRRL-BMS', 'UCSD-Folsom', 'SIRTA']
datasets = load_datasets(dataset_list)
print(datasets)
# you can check individual datasets
# e.g., show the structure of the dataset
print(datasets['SIRTA'])
# e.g., query specifc samples
display(datasets['SRRL-BMS']['train'][0]['image'])
Dataset Formatting¶
Sky images can be used for different tasks, such as solar forecasting, cloud segmentation, cloud classification, and cloud motion prediction. Below is an example utilizing the solar_forecast formatter.
formatted_datasets = FormatMLDataset.solar_forecast(datasets, forecast_span=2, # time span from the first to the last forecasts
forecast_resolution=1, # temporal resolution of the forecasts
lead_time=15, # the time between the forecast submission deadline and the first forecast timestamp
history_length=15, # time span from the first to the last input history
history_resolution=3, # temporal resolution of the input history
sample_interval=2, # the time interval between two consecutive samples
transform=None)
# Create validation splits for the formatted datasets
formatted_datasets.split("random", 0.8)
# Combine the formatted datasets into one through concatenation
combined_dataset = formatted_datasets.combine("concat")
print(f"Number of training samples: {len(combined_dataset['train'])}")
print(f"Number of validation samples: {len(combined_dataset['validation'])}")
print(f"Number of test samples: {len(combined_dataset['test'])}")
for _ in range(5):
idx = random.randint(0, len(combined_dataset['train']))
display_sample(combined_dataset['train'][idx])
Using a DataLoader¶
With a formatted dataset¶
# Can set a batch_size parameter to use batched elements for training
formatted_dataloader = get_dataloader(combined_dataset['train'], batch_size=2)
idx = 0
for example in formatted_dataloader:
display_sample(example, False, False)
idx += 1
if idx > 5:
break
Evaluating your Model¶
Provided are various functions for evaluation of a model's performance.
These include:
- Mean Absolute Error (MAE)
- Root Mean Squared Error (RMSE)
- Mean Bias Error (MBE)
- Relative RMSE (RRMSE)
- Temporal Distortion Index (TDI)
- Temporal Distortion Mix (TDM)
See Barhmi et. al. (2024) (https://www.mdpi.com/2673-9941/4/1/5) for more details.
# Generate some sample data
pred_ghi = np.array([random.randint(-5, 5) + pred for pred in [500, 500, 500, 500, 600, 700, 600, 500, 400, 400]])
target_ghi = np.array([500, 500, 600, 700, 600, 500, 400, 400, 400, 400])
plt.plot(pred_ghi, label='pred')
plt.plot(target_ghi, label='target')
plt.legend()
plt.show()
# Evaluate with the various metrics
print(f"MAE: {evaluation.mae(target_ghi, pred_ghi):.2f}")
print(f"RMSE: {evaluation.rmse(target_ghi, pred_ghi):.2f}")
print(f"MBE: {evaluation.mbe(target_ghi, pred_ghi):.2f}")
print(f"RRMSE: {evaluation.rrmse(target_ghi, pred_ghi):.2f}")
print(f"TDI: {evaluation.tdi(target_ghi, pred_ghi):.2f}")
print(f"TDI (Advance): {evaluation.tdi_adv(target_ghi, pred_ghi):.2f}")
print(f"TDI (Late): {evaluation.tdi_late(target_ghi, pred_ghi):.2f}")
print(f"TDM: {evaluation.tdm(target_ghi, pred_ghi):.2f}")
Loading a benchmark¶
benchmark = load_datasets('forecast-pv', num_proc=10)
print(benchmark)
Putting it all together¶
The following is an example combining the above steps into one cell.
datasets = ["SIPPMIF", "SKIPPD"]
datasets_container = load_datasets(datasets, num_proc=10)
formatted_datasets = FormatMLDataset.solar_estimation(datasets_container, target_label="pv", normalization="max")
formatted_datasets.split("random", 0.8)
combined_dataset = formatted_datasets.combine("concat")
train_dataloader = get_dataloader(combined_dataset['train'])
validation_dataloader = get_dataloader(combined_dataset['validation'])
test_dataloader = get_dataloader(combined_dataset['test'])
# Print out a couple samples from each loader
def print_non_image(sample):
labels = {key: value for key, value in sample.items() if key != 'image'}
print(f"Labels: {labels}")
itrain = iter(train_dataloader)
ivalid = iter(validation_dataloader)
itest = iter(test_dataloader)
for i in range(605):
train_sample = next(itrain)
validation_sample = next(ivalid)
test_sample = next(itest)
if i > 603: # Get samples from both datasets
print("Train Sample")
print("Prediction target:", train_sample['target'])
print_non_image(train_sample)
display_tensor_img(train_sample['image'][0])
print("Validation Sample")
print("Prediction target:", validation_sample['target'])
print_non_image(validation_sample)
display_tensor_img(validation_sample['image'][0])
print("Test Sample")
print("Prediction target:", test_sample['target'])
print_non_image(test_sample)
display_tensor_img(test_sample['image'][0])
