Process raw GPS data#
This notebook demonstrates the use of Smart Health modules to process synthetic raw GPS data ready for ingress to the “Intermediate” bucket.
# Import required libraries, modules and configurations
import logging
from pathlib import Path
from bhf_smarthealth.config import (
DEFAULT_TIME_PERIOD,
DEFAULT_REPLACE_NON_MATCHED_LSOA_VALUES,
)
from bhf_smarthealth.parse import gps_parser as bhf_gps
from bhf_smarthealth.utils import data_io, df_utils
# Initiate logging
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
# Paths are relative to this notebook
# Ouputs are saved to the repository
MOBILITY_FILEOUT_PATH = Path(
"../../../tests/data/intermediate/mobility_2025-10-09.parquet"
)
LSOA_FILEOUT_PATH = Path("../../../tests/data/intermediate/lsoa_2025-10-09.parquet")
# Path to the LSOA reference data
LSOA_path = Path("../../../tests/data/reference/lsoas_for_testing.pkl")
# Path to the raw (synthetic) GPS example data
data_source = Path("../../../tests/data/raw/UOS00001_20251009_gps.json")
Load raw JSON file
An information message shows how many valid data points have been loaded from the specified file
gps_json = bhf_gps.load_gps_json(data_source)
# Print the first row of the raw data
gps_json[0]
INFO:bhf_smarthealth.parse.gps_parser:Loaded 828 GPS data points from ../../../tests/data/raw/UOS00001_20251009_gps.json
{'d': '2025-10-09T00:00:00',
'lat': 53.3806626,
'long': -1.4702278,
'sheffield_id': 'UOS00001'}
The raw data file contents are validated using Pydantic
No error message here indicates that the file has passed validation checks that verify feasible lat/long values, that ID, datetime, lat, and long values are present for each timestamped datapoint, and that no extra, unexpected fields are present.
gps_val = bhf_gps.validate_gps_data(gps_json)
The JSON dictionary is converted to a GeoDataframe type
gps_gdf = bhf_gps.gps_to_dataframe(gps_val)
# Show the first five rows of the GeoDataFrame
gps_gdf.head()
| long | lat | sheffield_id | |
|---|---|---|---|
| timestamp | |||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 |
Convert the raw GPS latitude and longitude to geometry
gps_gdf = bhf_gps.add_gps_as_geometry(gps_gdf)
# Show the first five rows
gps_gdf.head()
| long | lat | sheffield_id | geometry | |
|---|---|---|---|---|
| timestamp | ||||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 | POINT (-1.47023 53.38066) |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 | POINT (-1.47022 53.3808) |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 | POINT (-1.47021 53.3809) |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 | POINT (-1.4702 53.38099) |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 | POINT (-1.47021 53.38105) |
Distances are calculated between each timepoint
gps_gdf["distance"] = bhf_gps.calculate_distances(gps_gdf)
# Show the first five rows
gps_gdf.head()
| long | lat | sheffield_id | geometry | distance | |
|---|---|---|---|---|---|
| timestamp | |||||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 | POINT (-1.47023 53.38066) | NaN |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 | POINT (-1.47022 53.3808) | 14.995 |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 | POINT (-1.47021 53.3809) | 11.996 |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 | POINT (-1.4702 53.38099) | 8.997 |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 | POINT (-1.47021 53.38105) | 6.998 |
Distance values are used to calculate the speed of travel between timepoints
gps_gdf["speed"] = bhf_gps.calculate_speeds(gps_gdf)
# Show the first five rows
gps_gdf.head()
| long | lat | sheffield_id | geometry | distance | speed | |
|---|---|---|---|---|---|---|
| timestamp | ||||||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 | POINT (-1.47023 53.38066) | NaN | NaN |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 | POINT (-1.47022 53.3808) | 14.995 | 1.0 |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 | POINT (-1.47021 53.3809) | 11.996 | 1.0 |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 | POINT (-1.4702 53.38099) | 8.997 | 1.0 |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 | POINT (-1.47021 53.38105) | 6.998 | 1.0 |
GPS data cleaning
Remove rows where speed between points is unreasonably large (by default > 36 metres per second)
Where there is a large time jump (by default greater than 5 minutes), set distance and speed to NULL so not included in calculations
gps_gdf_clean = df_utils.apply_func_list_to_df(
df=gps_gdf,
func_list=[
bhf_gps.filter_speeds,
bhf_gps.filter_time_jumps,
],
)
# Show the first five rows
gps_gdf_clean.head()
| long | lat | sheffield_id | geometry | distance | speed | |
|---|---|---|---|---|---|---|
| timestamp | ||||||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 | POINT (-1.47023 53.38066) | NaN | NaN |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 | POINT (-1.47022 53.3808) | 14.995 | 1.0 |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 | POINT (-1.47021 53.3809) | 11.996 | 1.0 |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 | POINT (-1.4702 53.38099) | 8.997 | 1.0 |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 | POINT (-1.47021 53.38105) | 6.998 | 1.0 |
Load the reference LSOA data and join to our GeoDataFrame
By default, the LSOA data is joined where the geometry point lies within an LSOA
lsoa_data = data_io.load_data_from_file(LSOA_path)
gps_lsoa_gdf = bhf_gps.join_spatial_data(
gps_gdf_clean, lsoa_data, nan_replace_values=DEFAULT_REPLACE_NON_MATCHED_LSOA_VALUES
)
gps_lsoa_gdf.head(10)
| long | lat | sheffield_id | geometry | distance | speed | index_right | LSOA21CD | LSOA21NM | |
|---|---|---|---|---|---|---|---|---|---|
| timestamp | |||||||||
| 2025-10-09 00:00:00 | -1.470228 | 53.380663 | UOS00001 | POINT (-1.47023 53.38066) | NaN | NaN | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:15 | -1.470220 | 53.380797 | UOS00001 | POINT (-1.47022 53.3808) | 14.995 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:27 | -1.470207 | 53.380905 | UOS00001 | POINT (-1.47021 53.3809) | 11.996 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:36 | -1.470205 | 53.380986 | UOS00001 | POINT (-1.4702 53.38099) | 8.997 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:43 | -1.470207 | 53.381049 | UOS00001 | POINT (-1.47021 53.38105) | 6.998 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:50 | -1.470205 | 53.381111 | UOS00001 | POINT (-1.4702 53.38111) | 6.998 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:57 | -1.470194 | 53.381174 | UOS00001 | POINT (-1.47019 53.38117) | 6.998 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:01:07 | -1.470184 | 53.381264 | UOS00001 | POINT (-1.47018 53.38126) | 9.996 | 1.0 | 28.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:07:40 | -1.469874 | 53.384790 | UOS00001 | POINT (-1.46987 53.38479) | NaN | NaN | 27.0 | E01033262 | Sheffield 073B |
| 2025-10-09 00:07:50 | -1.469856 | 53.384879 | UOS00001 | POINT (-1.46986 53.38488) | 9.996 | 1.0 | 27.0 | E01033262 | Sheffield 073B |
Remove unnecessary columns
This removes sensitive GPS data, and any join artefacts from combining with the LSOA reference data
# List of column names to be removed
columns = ["lat", "long", "geometry", "index_right"]
lsoa_gdf = df_utils.remove_columns(gps_lsoa_gdf, col_names=columns)
# Show first five rows
lsoa_gdf.head()
| sheffield_id | distance | speed | LSOA21CD | LSOA21NM | |
|---|---|---|---|---|---|
| timestamp | |||||
| 2025-10-09 00:00:00 | UOS00001 | NaN | NaN | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:15 | UOS00001 | 14.995 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:27 | UOS00001 | 11.996 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:36 | UOS00001 | 8.997 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:43 | UOS00001 | 6.998 | 1.0 | E01033264 | Sheffield 073D |
Set LSOA values to be NULL at bad timepoints, i.e. where the distance is NULL
This is done separately to the time jump filter so that the LSOA values can be kept if required.
lsoa_gdf_clean = bhf_gps.nullify_lsoa_at_invalid_timepoints(lsoa_gdf)
# Show first five rows
lsoa_gdf_clean.head(10)
| sheffield_id | distance | speed | LSOA21CD | LSOA21NM | |
|---|---|---|---|---|---|
| timestamp | |||||
| 2025-10-09 00:00:15 | UOS00001 | 14.995 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:27 | UOS00001 | 11.996 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:36 | UOS00001 | 8.997 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:43 | UOS00001 | 6.998 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:50 | UOS00001 | 6.998 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:00:57 | UOS00001 | 6.998 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:01:07 | UOS00001 | 9.996 | 1.0 | E01033264 | Sheffield 073D |
| 2025-10-09 00:07:50 | UOS00001 | 9.996 | 1.0 | E01033262 | Sheffield 073B |
| 2025-10-09 00:07:57 | UOS00001 | 6.998 | 1.0 | E01033262 | Sheffield 073B |
| 2025-10-09 00:08:03 | UOS00001 | 5.998 | 1.0 | E01033262 | Sheffield 073B |
Aggregate data into time windows
The latest LSOA in the time window is selected. If the latest LSOA value in the time window is associated with poor quality data (e.g. after a time jump) then the most recent LSOA is taken, moving backwards.
# Specify the calculations to perform on each aggregated field
agg_dict = {
"distance": ["size", "sum"],
"speed": ["mean"],
"LSOA21CD": ["last"],
"LSOA21NM": ["last", "nunique"],
}
# Aggregate into, by default, 5 minute intervals and apply the above aggregate calculations
lsoa_agg = df_utils.apply_agg_over_interval(
lsoa_gdf_clean,
agg_dict=agg_dict,
time_period=DEFAULT_TIME_PERIOD,
ID_col="sheffield_id",
)
# Show the first five rows
lsoa_agg.head()
| distance_size | distance_sum | speed_mean | LSOA21CD_last | LSOA21NM_last | LSOA21NM_nunique | ||
|---|---|---|---|---|---|---|---|
| sheffield_id | timestamp | ||||||
| UOS00001 | 2025-10-09 00:00:00 | 7 | 66.978 | 1.0 | E01033264 | Sheffield 073D | 1 |
| 2025-10-09 00:05:00 | 4 | 31.989 | 1.0 | E01033270 | Sheffield 022H | 2 | |
| 2025-10-09 00:55:00 | 17 | 172.938 | 1.0 | E01008063 | Sheffield 011B | 2 | |
| 2025-10-09 01:40:00 | 1 | 14.995 | 1.0 | E01008138 | Sheffield 006D | 1 | |
| 2025-10-09 01:55:00 | 9 | 92.967 | 1.0 | E01007891 | Sheffield 004D | 1 |
Rename columns
lsoa_agg_renamed = df_utils.rename_columns(
df=lsoa_agg,
col_names=[
"n_samples",
"dist_travelled",
"mean_speed",
"latest_LSOA21CD",
"latest_LSOA21NM",
"n_unique_LSOAs",
],
)
# Show the first five rows
lsoa_agg_renamed.head()
| n_samples | dist_travelled | mean_speed | latest_LSOA21CD | latest_LSOA21NM | n_unique_LSOAs | ||
|---|---|---|---|---|---|---|---|
| sheffield_id | timestamp | ||||||
| UOS00001 | 2025-10-09 00:00:00 | 7 | 66.978 | 1.0 | E01033264 | Sheffield 073D | 1 |
| 2025-10-09 00:05:00 | 4 | 31.989 | 1.0 | E01033270 | Sheffield 022H | 2 | |
| 2025-10-09 00:55:00 | 17 | 172.938 | 1.0 | E01008063 | Sheffield 011B | 2 | |
| 2025-10-09 01:40:00 | 1 | 14.995 | 1.0 | E01008138 | Sheffield 006D | 1 | |
| 2025-10-09 01:55:00 | 9 | 92.967 | 1.0 | E01007891 | Sheffield 004D | 1 |
Save data
This is the form the data will be in within the “Intermediate” bucket
mobility_df = lsoa_agg_renamed[
["dist_travelled", "mean_speed", "n_samples", "n_unique_LSOAs"]
]
lsoa_df = lsoa_agg_renamed[["n_samples", "latest_LSOA21NM", "latest_LSOA21CD"]]
mobility_df = mobility_df.rename(columns={"n_samples": "mobility_n_samples"})
lsoa_df = lsoa_df.rename(columns={"n_samples": "LSOA_n_samples"})
data_io.save_dataframe_ouput(mobility_df, MOBILITY_FILEOUT_PATH)
data_io.save_dataframe_ouput(lsoa_df, LSOA_FILEOUT_PATH)
Re-read in the data to check its integrity
intermediate_mobility_df = data_io.load_data_from_file(MOBILITY_FILEOUT_PATH)
intermediate_mobility_df.head()
| dist_travelled | mean_speed | mobility_n_samples | n_unique_LSOAs | ||
|---|---|---|---|---|---|
| sheffield_id | timestamp | ||||
| UOS00001 | 2025-10-09 00:00:00 | 66.978 | 1.0 | 7 | 1 |
| 2025-10-09 00:05:00 | 31.989 | 1.0 | 4 | 2 | |
| 2025-10-09 00:55:00 | 172.938 | 1.0 | 17 | 2 | |
| 2025-10-09 01:40:00 | 14.995 | 1.0 | 1 | 1 | |
| 2025-10-09 01:55:00 | 92.967 | 1.0 | 9 | 1 |
intermediate_lsoa_df = data_io.load_data_from_file(LSOA_FILEOUT_PATH)
intermediate_lsoa_df.head()
| LSOA_n_samples | latest_LSOA21NM | latest_LSOA21CD | ||
|---|---|---|---|---|
| sheffield_id | timestamp | |||
| UOS00001 | 2025-10-09 00:00:00 | 7 | Sheffield 073D | E01033264 |
| 2025-10-09 00:05:00 | 4 | Sheffield 022H | E01033270 | |
| 2025-10-09 00:55:00 | 17 | Sheffield 011B | E01008063 | |
| 2025-10-09 01:40:00 | 1 | Sheffield 006D | E01008138 | |
| 2025-10-09 01:55:00 | 9 | Sheffield 004D | E01007891 |