Introduction

In this notebook I implement a neural network based solution for building footprint detection on the SpaceNet7 dataset. I ignore the temporal aspect of the orginal challenge and focus on performing segmentation to detect buildings on single images. I use fastai, a deep learning library based on PyTorch. It provides functionality to train neural networks with modern best practices while reducing the amount of boilerplate code required.

Dataset Downloading

from google.colab import drive
drive.mount('/content/gdrive')
Mounted at /content/gdrive
cd /content/gdrive/Shareddrives/Undrive
/content/gdrive/Shareddrives/Undrive
# !unzip spacenet-7-multitemporal-urban-development.zip -d s7

START

The dataset is stored on AWS. Instructions how to install are here.

Installing Libraries and Preparing requirements.txt for reproducbillity

 
!pip freeze > requirements.txt
ls
models/           s7/                                             wandb/
requirements.txt  spacenet-7-multitemporal-urban-development.zip

Setup

cd /content/gdrive/Shareddrives/Undrive/s7/SN7_buildings_train/train/
/content/gdrive/Shareddrives/Undrive/s7/SN7_buildings_train/train
 
!ls
L15-0331E-1257N_1327_3160_13  L15-1203E-1203N_4815_3378_13
L15-0357E-1223N_1429_3296_13  L15-1204E-1202N_4816_3380_13
L15-0358E-1220N_1433_3310_13  L15-1204E-1204N_4819_3372_13
L15-0361E-1300N_1446_2989_13  L15-1209E-1113N_4838_3737_13
L15-0368E-1245N_1474_3210_13  L15-1210E-1025N_4840_4088_13
L15-0387E-1276N_1549_3087_13  L15-1276E-1107N_5105_3761_13
L15-0434E-1218N_1736_3318_13  L15-1289E-1169N_5156_3514_13
L15-0457E-1135N_1831_3648_13  L15-1296E-1198N_5184_3399_13
L15-0487E-1246N_1950_3207_13  L15-1298E-1322N_5193_2903_13
L15-0506E-1204N_2027_3374_13  L15-1335E-1166N_5342_3524_13
L15-0544E-1228N_2176_3279_13  L15-1389E-1284N_5557_3054_13
L15-0566E-1185N_2265_3451_13  L15-1438E-1134N_5753_3655_13
L15-0571E-1075N_2287_3888_13  L15-1439E-1134N_5759_3655_13
L15-0577E-1243N_2309_3217_13  L15-1479E-1101N_5916_3785_13
L15-0586E-1127N_2345_3680_13  L15-1481E-1119N_5927_3715_13
L15-0595E-1278N_2383_3079_13  L15-1538E-1163N_6154_3539_13
L15-0614E-0946N_2459_4406_13  L15-1615E-1205N_6460_3370_13
L15-0632E-0892N_2528_4620_13  L15-1615E-1206N_6460_3366_13
L15-0683E-1006N_2732_4164_13  L15-1617E-1207N_6468_3360_13
L15-0760E-0887N_3041_4643_13  L15-1669E-1153N_6678_3579_13
L15-0924E-1108N_3699_3757_13  L15-1669E-1160N_6678_3548_13
L15-0977E-1187N_3911_3441_13  L15-1669E-1160N_6679_3549_13
L15-1014E-1375N_4056_2688_13  L15-1672E-1207N_6691_3363_13
L15-1015E-1062N_4061_3941_13  L15-1690E-1211N_6763_3346_13
L15-1025E-1366N_4102_2726_13  L15-1691E-1211N_6764_3347_13
L15-1049E-1370N_4196_2710_13  L15-1703E-1219N_6813_3313_13
L15-1138E-1216N_4553_3325_13  L15-1709E-1112N_6838_3742_13
L15-1172E-1306N_4688_2967_13  L15-1716E-1211N_6864_3345_13
L15-1185E-0935N_4742_4450_13  models
L15-1200E-0847N_4802_4803_13  wandb
 
L15-0331E-1257N_1327_3160_13  L15-1200E-0847N_4802_4803_13
L15-0357E-1223N_1429_3296_13  L15-1203E-1203N_4815_3378_13
L15-0358E-1220N_1433_3310_13  L15-1204E-1202N_4816_3380_13
L15-0361E-1300N_1446_2989_13  L15-1204E-1204N_4819_3372_13
L15-0368E-1245N_1474_3210_13  L15-1209E-1113N_4838_3737_13
L15-0387E-1276N_1549_3087_13  L15-1210E-1025N_4840_4088_13
L15-0434E-1218N_1736_3318_13  L15-1276E-1107N_5105_3761_13
L15-0457E-1135N_1831_3648_13  L15-1289E-1169N_5156_3514_13
L15-0487E-1246N_1950_3207_13  L15-1296E-1198N_5184_3399_13
L15-0506E-1204N_2027_3374_13  L15-1298E-1322N_5193_2903_13
L15-0544E-1228N_2176_3279_13  L15-1335E-1166N_5342_3524_13
L15-0566E-1185N_2265_3451_13  L15-1389E-1284N_5557_3054_13
L15-0571E-1075N_2287_3888_13  L15-1438E-1134N_5753_3655_13
L15-0577E-1243N_2309_3217_13  L15-1439E-1134N_5759_3655_13
L15-0586E-1127N_2345_3680_13  L15-1479E-1101N_5916_3785_13
L15-0595E-1278N_2383_3079_13  L15-1481E-1119N_5927_3715_13
L15-0614E-0946N_2459_4406_13  L15-1538E-1163N_6154_3539_13
L15-0632E-0892N_2528_4620_13  L15-1615E-1205N_6460_3370_13
L15-0683E-1006N_2732_4164_13  L15-1615E-1206N_6460_3366_13
L15-0760E-0887N_3041_4643_13  L15-1617E-1207N_6468_3360_13
L15-0924E-1108N_3699_3757_13  L15-1669E-1153N_6678_3579_13
L15-0977E-1187N_3911_3441_13  L15-1669E-1160N_6678_3548_13
L15-1014E-1375N_4056_2688_13  L15-1669E-1160N_6679_3549_13
L15-1015E-1062N_4061_3941_13  L15-1672E-1207N_6691_3363_13
L15-1025E-1366N_4102_2726_13  L15-1690E-1211N_6763_3346_13
L15-1049E-1370N_4196_2710_13  L15-1691E-1211N_6764_3347_13
L15-1138E-1216N_4553_3325_13  L15-1703E-1219N_6813_3313_13
L15-1172E-1306N_4688_2967_13  L15-1709E-1112N_6838_3742_13
L15-1185E-0935N_4742_4450_13  L15-1716E-1211N_6864_3345_13
path.ls()
(#59) [Path('L15-0331E-1257N_1327_3160_13'),Path('L15-0357E-1223N_1429_3296_13'),Path('L15-0358E-1220N_1433_3310_13'),Path('L15-0361E-1300N_1446_2989_13'),Path('L15-0368E-1245N_1474_3210_13'),Path('L15-0387E-1276N_1549_3087_13'),Path('L15-0434E-1218N_1736_3318_13'),Path('L15-0457E-1135N_1831_3648_13'),Path('L15-0487E-1246N_1950_3207_13'),Path('L15-0506E-1204N_2027_3374_13')...]

Defining training parameters:

cd /content/gdrive/Shareddrives/Undrive/s7/SN7_buildings_train/train
/content/gdrive/Shareddrives/Undrive/s7/SN7_buildings_train/train
ls
L15-0331E-1257N_1327_3160_13/  L15-1200E-0847N_4802_4803_13/
L15-0357E-1223N_1429_3296_13/  L15-1203E-1203N_4815_3378_13/
L15-0358E-1220N_1433_3310_13/  L15-1204E-1202N_4816_3380_13/
L15-0361E-1300N_1446_2989_13/  L15-1204E-1204N_4819_3372_13/
L15-0368E-1245N_1474_3210_13/  L15-1209E-1113N_4838_3737_13/
L15-0387E-1276N_1549_3087_13/  L15-1210E-1025N_4840_4088_13/
L15-0434E-1218N_1736_3318_13/  L15-1276E-1107N_5105_3761_13/
L15-0457E-1135N_1831_3648_13/  L15-1289E-1169N_5156_3514_13/
L15-0487E-1246N_1950_3207_13/  L15-1296E-1198N_5184_3399_13/
L15-0506E-1204N_2027_3374_13/  L15-1298E-1322N_5193_2903_13/
L15-0544E-1228N_2176_3279_13/  L15-1335E-1166N_5342_3524_13/
L15-0566E-1185N_2265_3451_13/  L15-1389E-1284N_5557_3054_13/
L15-0571E-1075N_2287_3888_13/  L15-1438E-1134N_5753_3655_13/
L15-0577E-1243N_2309_3217_13/  L15-1439E-1134N_5759_3655_13/
L15-0586E-1127N_2345_3680_13/  L15-1479E-1101N_5916_3785_13/
L15-0595E-1278N_2383_3079_13/  L15-1481E-1119N_5927_3715_13/
L15-0614E-0946N_2459_4406_13/  L15-1538E-1163N_6154_3539_13/
L15-0632E-0892N_2528_4620_13/  L15-1615E-1205N_6460_3370_13/
L15-0683E-1006N_2732_4164_13/  L15-1615E-1206N_6460_3366_13/
L15-0760E-0887N_3041_4643_13/  L15-1617E-1207N_6468_3360_13/
L15-0924E-1108N_3699_3757_13/  L15-1669E-1153N_6678_3579_13/
L15-0977E-1187N_3911_3441_13/  L15-1669E-1160N_6678_3548_13/
L15-1014E-1375N_4056_2688_13/  L15-1669E-1160N_6679_3549_13/
L15-1015E-1062N_4061_3941_13/  L15-1672E-1207N_6691_3363_13/
L15-1025E-1366N_4102_2726_13/  L15-1690E-1211N_6763_3346_13/
L15-1049E-1370N_4196_2710_13/  L15-1691E-1211N_6764_3347_13/
L15-1138E-1216N_4553_3325_13/  L15-1703E-1219N_6813_3313_13/
L15-1172E-1306N_4688_2967_13/  L15-1709E-1112N_6838_3742_13/
L15-1185E-0935N_4742_4450_13/  L15-1716E-1211N_6864_3345_13/
BATCH_SIZE = 12 #   (3 for xresnet50, 12 for xresnet34 with Tesla P100/T4)
TILES_PER_SCENE = 16
ARCHITECTURE = xresnet34
EPOCHS = 40
CLASS_WEIGHTS = [0.25,0.75]
LR_MAX = 3e-4
ENCODER_FACTOR = 10
CODES = ['Land','Building']
# Weights and Biases config
config_dictionary = dict(
    bs=BATCH_SIZE,
    tiles_per_scene=TILES_PER_SCENE,
    architecture = str(ARCHITECTURE),
    epochs = EPOCHS,
    class_weights = CLASS_WEIGHTS,
    lr_max = LR_MAX,
    encoder_factor = ENCODER_FACTOR
)
BATCH_SIZE = 12 # 3 for xresnet50, 12 for xresnet34 with Tesla P100 (16GB)
TILES_PER_SCENE = 16
ARCHITECTURE = xresnet34
EPOCHS = 80
CLASS_WEIGHTS = [0.25,0.75]
LR_MAX = 3e-4
ENCODER_FACTOR = 10
CODES = ['Land','Building']
BATCH_SIZE = 3 # 3 for xresnet50, 12 for xresnet34 with Tesla P100 (16GB)
TILES_PER_SCENE = 16
ARCHITECTURE = xresnet50
EPOCHS = 40
CLASS_WEIGHTS = [0.25,0.75]
LR_MAX = 3e-4
ENCODER_FACTOR = 10
CODES = ['Land','Building']
!ls
models		  s7						  wandb
requirements.txt  spacenet-7-multitemporal-urban-development.zip

Data-Preprocessing

Exploring dataset structure, display sample scene directories:

!nvidia-smi
Tue Jul  6 07:44:58 2021       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.27       Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   37C    P8     9W /  70W |      3MiB / 15109MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

scenes = path.ls().sorted()
print(f'Numer of scenes: {len(scenes)}')
pprint(list(scenes)[:5])
Numer of scenes: 59
[Path('.ipynb_checkpoints'),
 Path('L15-0331E-1257N_1327_3160_13'),
 Path('L15-0357E-1223N_1429_3296_13'),
 Path('L15-0358E-1220N_1433_3310_13'),
 Path('L15-0361E-1300N_1446_2989_13')]

Which folders are in each scene (the last three have been added later during processing)

sample_scene = (path/'L15-0683E-1006N_2732_4164_13')
pprint(list(sample_scene.ls()))
[Path('L15-0683E-1006N_2732_4164_13/UDM_masks'),
 Path('L15-0683E-1006N_2732_4164_13/images'),
 Path('L15-0683E-1006N_2732_4164_13/images_masked'),
 Path('L15-0683E-1006N_2732_4164_13/labels'),
 Path('L15-0683E-1006N_2732_4164_13/labels_match'),
 Path('L15-0683E-1006N_2732_4164_13/labels_match_pix'),
 Path('L15-0683E-1006N_2732_4164_13/binary_mask'),
 Path('L15-0683E-1006N_2732_4164_13/img_tiles'),
 Path('L15-0683E-1006N_2732_4164_13/mask_tiles')]

How many images are in a specific scene:

images_masked = (sample_scene/'images_masked').ls().sorted()
labels = (sample_scene/'labels_match').ls().sorted()
print(f'Numer of images in scene: {len(images_masked)}')
pprint(list(images_masked[:5]))
Numer of images in scene: 22
[Path('L15-0683E-1006N_2732_4164_13/images_masked/global_monthly_2018_01_mosaic_L15-0683E-1006N_2732_4164_13.tif'),
 Path('L15-0683E-1006N_2732_4164_13/images_masked/global_monthly_2018_02_mosaic_L15-0683E-1006N_2732_4164_13.tif'),
 Path('L15-0683E-1006N_2732_4164_13/images_masked/global_monthly_2018_03_mosaic_L15-0683E-1006N_2732_4164_13.tif'),
 Path('L15-0683E-1006N_2732_4164_13/images_masked/global_monthly_2018_04_mosaic_L15-0683E-1006N_2732_4164_13.tif'),
 Path('L15-0683E-1006N_2732_4164_13/images_masked/global_monthly_2018_06_mosaic_L15-0683E-1006N_2732_4164_13.tif')]

There are 58 scenes of 4km x 4km in the dataset, each containing about 24 images over the span of two years.

Let's pick one example image and its polygons:

image, shapes = images_masked[0], labels[0]

We use the images that have UDM masks where clouds were in the original picture:

show_image(PILImage.create(image), figsize=(12,12));

Creating binary masks

This is a function to generate binary mask images from geojson vector files. Source

import rasterio
from rasterio.plot import reshape_as_image
import rasterio.mask
from rasterio.features import rasterize

import pandas as pd
import geopandas as gpd
from shapely.geometry import mapping, Point, Polygon
from shapely.ops import cascaded_union

# SOURCE:  https://lpsmlgeo.github.io/2019-09-22-binary_mask/

def generate_mask(raster_path, shape_path, output_path=None, file_name=None):

    """Function that generates a binary mask from a vector file (shp or geojson)
    raster_path = path to the .tif;
    shape_path = path to the shapefile or GeoJson.
    output_path = Path to save the binary mask.
    file_name = Name of the file.
    """
    
    #load raster
    
    with rasterio.open(raster_path, "r") as src:
        raster_img = src.read()
        raster_meta = src.meta
    
    #load o shapefile ou GeoJson
    train_df = gpd.read_file(shape_path)
    
    #Verify crs
    if train_df.crs != src.crs:
        print(" Raster crs : {}, Vector crs : {}.\n Convert vector and raster to the same CRS.".format(src.crs,train_df.crs))
        
        
    #Function that generates the mask
    def poly_from_utm(polygon, transform):
        poly_pts = []

        poly = cascaded_union(polygon)
        for i in np.array(poly.exterior.coords):

            poly_pts.append(~transform * tuple(i))

        new_poly = Polygon(poly_pts)
        return new_poly
    
    
    poly_shp = []
    im_size = (src.meta['height'], src.meta['width'])
    for num, row in train_df.iterrows():
        if row['geometry'].geom_type == 'Polygon':
            poly = poly_from_utm(row['geometry'], src.meta['transform'])
            poly_shp.append(poly)
        else:
            for p in row['geometry']:
                poly = poly_from_utm(p, src.meta['transform'])
                poly_shp.append(poly)

    #set_trace()
    
    if len(poly_shp) > 0:
      mask = rasterize(shapes=poly_shp,
                      out_shape=im_size)
    else:
      mask = np.zeros(im_size)
    
    # Save or show mask
    mask = mask.astype("uint8")    
    bin_mask_meta = src.meta.copy()
    bin_mask_meta.update({'count': 1})
    if (output_path != None and file_name != None):
      os.chdir(output_path)
      with rasterio.open(file_name, 'w', **bin_mask_meta) as dst:
          dst.write(mask * 255, 1)
    else: 
      return mask

Show a mask:

mask = generate_mask(image, shapes)
plt.figure(figsize=(12,12))
plt.tight_layout()
plt.xticks([])
plt.yticks([])
plt.imshow(mask,cmap='cividis');