Mohammad’s Blog - Recognizing Hand written Digits with deep learning on Kaggle full dataset

0.1 We are importing all the required packages and data

We will use fast AI for our production. Fast AI is a framework on top of pytorch which makes our development of deep learning models easy

import numpy as np
import pandas as pd


#hide
!pip install -Uqq fastbook
import fastbook
fastbook.setup_book()

ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
jupyterlab-git 0.11.0 requires nbdime<2.0.0,>=1.1.0, but you have nbdime 2.1.0 which is incompatible.

#hide
from fastai.vision.all import *
from fastbook import *

matplotlib.rc('image', cmap='Greys')

from fastai import *
from fastai.vision import *

# to get all files from a directory
import os

# to easier work with paths
from pathlib import Path

# to read and manipulate .csv-files
import pandas as pd

INPUT = Path("../input/digit-recognizer")

0.2 We could see our data

train_df = pd.read_csv(INPUT/"train.csv")
train_df.head(3)

	label	pixel122	pixel123	pixel124	pixel125	pixel126	pixel127	pixel128	pixel129	pixel132	pixel133	pixel134	pixel148	pixel149	pixel150	pixel151	pixel152	pixel153	pixel154	pixel155	pixel156	pixel157	pixel158	pixel159	pixel160	pixel161	pixel162	pixel175	pixel176	pixel177	pixel178	pixel179	pixel180	pixel181	pixel182	pixel183	pixel184	pixel185	pixel186	pixel187	pixel188	pixel189	pixel190	pixel203	pixel204	pixel205	pixel206	pixel207	pixel208	pixel209	pixel210	pixel211	pixel212	pixel213	pixel214	pixel215	pixel216	pixel217	pixel230	pixel231	pixel232	pixel233	pixel234	pixel235	pixel236	pixel237	pixel238	pixel239	pixel240	pixel241	pixel242	pixel243	pixel244	pixel245	pixel258	pixel259	pixel260	pixel261	pixel262	pixel263	pixel264	pixel265	pixel266	pixel267	pixel268	pixel269	pixel270	pixel271	pixel272	pixel273	pixel285	pixel286	pixel287	pixel288	pixel289	pixel290	pixel292	pixel293	pixel294	pixel295	pixel296	pixel297	pixel298	pixel299	pixel300	pixel301	pixel302	pixel313	pixel314	pixel315	pixel316	pixel317	pixel320	pixel321	pixel322	pixel323	pixel324	pixel325	pixel326	pixel327	pixel328	pixel329	pixel330	pixel341	pixel342	pixel343	pixel344	pixel345	pixel348	pixel349	pixel350	pixel351	pixel352	pixel353	pixel354	pixel355	pixel356	pixel357	pixel358	pixel369	pixel370	pixel371	pixel372	pixel373	pixel376	pixel377	pixel378	pixel379	pixel380	pixel383	pixel384	pixel385	pixel386	pixel387	pixel396	pixel397	pixel398	pixel399	pixel400	pixel401	pixel403	pixel404	pixel405	pixel406	pixel407	pixel408	pixel411	pixel412	pixel413	pixel414	pixel415	pixel424	pixel425	pixel426	pixel427	pixel428	pixel429	pixel431	pixel432	pixel433	pixel434	pixel435	pixel439	pixel440	pixel441	pixel442	pixel443	pixel452	pixel453	pixel454	pixel455	pixel456	pixel457	pixel458	pixel459	pixel460	pixel461	pixel462	pixel463	pixel467	pixel468	pixel469	pixel470	pixel471	pixel481	pixel482	pixel483	pixel484	pixel485	pixel486	pixel487	pixel488	pixel489	pixel490	pixel491	pixel494	pixel495	pixel496	pixel497	pixel498	pixel499	pixel509	pixel510	pixel511	pixel512	pixel513	pixel514	pixel515	pixel516	pixel517	pixel518	pixel519	pixel521	pixel522	pixel523	pixel524	pixel525	pixel526	pixel538	pixel539	pixel540	pixel541	pixel542	pixel543	pixel544	pixel545	pixel546	pixel547	pixel548	pixel549	pixel550	pixel551	pixel552	pixel553	pixel567	pixel568	pixel569	pixel570	pixel571	pixel572	pixel573	pixel574	pixel575	pixel576	pixel577	pixel578	pixel579	pixel580	pixel581	pixel595	pixel596	pixel597	pixel598	pixel599	pixel600	pixel601	pixel602	pixel603	pixel604	pixel605	pixel606	pixel607	pixel608	pixel609	pixel623	pixel624	pixel625	pixel626	pixel627	pixel628	pixel629	pixel630	pixel631	pixel632	pixel633	pixel634	pixel635	pixel652	pixel653	pixel654	pixel655	pixel656	pixel657	pixel658	pixel659	pixel660	pixel661	pixel662
0	1	0	0	0	0	0	0	0	0	188	255	94	0	0	0	0	0	0	0	0	0	0	0	191	250	253	93	0	0	0	0	0	0	0	0	0	0	0	123	248	253	167	10	0	0	0	0	0	0	0	0	0	0	80	247	253	208	13	0	0	0	0	0	0	0	0	0	0	29	207	253	235	77	0	0	0	0	0	0	0	0	0	0	54	209	253	253	88	0	0	0	0	0	0	0	0	0	0	93	254	253	238	170	17	0	0	0	0	0	0	0	0	0	23	210	254	253	159	0	0	0	0	0	0	0	0	0	0	16	209	253	254	240	81	0	0	0	0	0	0	0	0	0	0	27	253	253	254	13	0	0	0	0	0	0	0	0	0	0	0	20	206	254	254	198	7	0	0	0	0	0	0	0	0	0	0	0	168	253	253	196	7	0	0	0	0	0	0	0	0	0	0	0	20	203	253	248	76	0	0	0	0	0	0	0	0	0	0	22	188	253	245	93	0	0	0	0	0	0	0	0	0	0	0	0	103	253	253	191	0	0	0	0	0	0	0	0	0	0	0	89	240	253	195	25	0	0	0	0	0	0	0	0	0	15	220	253	253	80	0	0	0	0	0	0	0	0	0	0	94	253	253	253	94	0	0	0	0	0	0	0	0	0	0	89	251	253	250	131	0	0	0	0	0	0	0	0	214	218	95	0	0	0	0	0	0	0	0
1	0	18	30	137	137	192	86	72	1	0	0	0	13	86	250	254	254	254	254	217	246	151	32	0	0	0	0	16	179	254	254	254	254	254	254	254	254	254	231	54	15	0	0	72	254	254	254	254	254	254	254	254	254	254	254	254	104	0	61	191	254	254	254	254	254	109	83	199	254	254	254	254	243	85	172	254	254	254	202	147	147	45	0	11	29	200	254	254	254	171	1	174	254	254	89	67	0	0	0	0	0	128	252	254	254	212	76	47	254	254	254	29	0	0	0	0	0	0	83	254	254	254	153	80	254	254	240	24	0	0	0	0	0	0	25	240	254	254	153	64	254	254	186	7	0	0	0	0	0	166	254	254	224	12	14	232	254	254	254	29	0	0	0	0	0	0	75	254	254	254	17	18	254	254	254	254	29	0	0	0	0	0	48	254	254	254	17	2	163	254	254	254	29	0	0	0	0	0	0	48	254	254	254	17	94	254	254	254	200	12	0	0	0	0	0	16	209	254	254	150	1	15	206	254	254	254	202	66	0	0	0	0	21	161	254	254	245	31	60	212	254	254	254	194	48	48	34	41	48	209	254	254	254	171	86	243	254	254	254	254	254	233	243	254	254	254	254	254	86	0	114	254	254	254	254	254	254	254	254	254	254	239	86	11	0	13	182	254	254	254	254	254	254	254	254	243	70	0	8	76	146	254	255	254	255	146	19	15
2	1	0	0	3	141	139	3	0	0	0	0	0	0	0	0	0	9	254	254	8	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	8	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	106	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	6	185	254	184	0	0	0	0	0	0	0	0	0	0	0	0	0	89	254	184	0	0	0	0	0	0	0	0	0	0	0	0	4	146	254	184	0	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	0	9	254	254	184	0	0	0	0	0	0	0	0	0	0	0	0	156	254	254	184	0	0	0	0	0	0	0	0	0	0	0	185	255	255	184	0	0	0	0	0	0	0	0	0	0	0	185	254	254	184	0	0	0	0	0	0	0	0	0	0	0	185	254	254	184	0	0	0	0	0	0	0	0	63	254	254	62	0	0	0

test_df = pd.read_csv(INPUT/"test.csv")
test_df.head(3)
test_df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 28000 entries, 0 to 27999
Columns: 784 entries, pixel0 to pixel783
dtypes: int64(784)
memory usage: 167.5 MB

TRAIN = Path("../train")
TEST = Path("../test")

0.2.1 As fast AI deep learning models doesn’t work on Dataframe. We will make a directory consisting of each number (0 to 9).

# Create training directory
for index in range(10):
    try:
        os.makedirs(TRAIN/str(index))
    except:
        pass

#Create test directory
try:
    os.makedirs(TEST)
except:
    pass

0.3 Now we will convert our Dataframe which consist of 785 columns where one column is the label of the digit and all the other columns are the values of pixels (28 * 28) =784 pixels

We will convert our pixels into images

# import numpy to reshape array from flat (1x784) to square (28x28)
import numpy as np

# import PIL to display images and to create images from arrays
from PIL import Image

def saveDigit(digit, filepath):
    digit = digit.reshape(28,28)
    digit = digit.astype(np.uint8)

    img = Image.fromarray(digit)
    img.save(filepath)

# save training images
for index, row in train_df.iterrows():
    
    label,digit = row[0], row[1:]
    
    folder = TRAIN/str(label)
    filename = f"{index}.jpg"
    filepath = folder/filename
    
    digit = digit.values
    
    saveDigit(digit, filepath)

# save testing images
for index, digit in test_df.iterrows():

    folder = TEST
    filename = f"{index}.jpg"
    filepath = folder/filename
    
    digit = digit.values
    
    saveDigit(digit, filepath)

Let’s see one of our images

subdirectory=str(0)
path = TEST
images = os.listdir(path)
image = Image.open(path/images[2])
image

0.4 See how each image is mapped in terms of pixels

import matplotlib.pyplot as plt
image_path = TEST/os.listdir(TEST)[9]
image = Image.open(image_path)
image_array = np.asarray(image)


fig, ax = plt.subplots(figsize=(15, 15))

img = ax.imshow(image_array, cmap='gray')

for x in range(28):
    for y in range(28):
        value = round(image_array[y][x]/255.0, 2)
        color = 'black' if value > 0.5 else 'white'
        ax.annotate(text=value, xy=(x, y), ha='center', va='center', color=color)

plt.axis('off')
plt.show()

0.5 Let’s make our dataloaders

data = ImageDataLoaders.from_folder(
    path = TRAIN,  
    valid_pct = 0.1,
    bs = 256,
    size = 28,

    
)

1 We are using here resnet50 modal and freezed model and trained last layer for three epochs then unfreeze layers to train our model

from fastai.callback.fp16 import *
learn = cnn_learner(data, resnet50, metrics=accuracy).to_fp16()
learn.fine_tune(12, freeze_epochs=3)

Downloading: "https://download.pytorch.org/models/resnet50-19c8e357.pth" to /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth

epoch	train_loss	valid_loss	accuracy	time
0	1.295694	0.723254	0.770952	00:47
1	0.795691	0.495572	0.839286	00:45
2	0.495209	0.346795	0.891667	00:46

epoch	train_loss	valid_loss	accuracy	time
0	0.195334	0.132666	0.958333	00:48
1	0.085935	0.087034	0.973571	00:49
2	0.056619	0.070643	0.979048	00:48
3	0.043681	0.057061	0.983571	00:49
4	0.024404	0.061974	0.984524	00:48
5	0.018393	0.057527	0.985952	00:48
6	0.013382	0.040939	0.988810	00:48
7	0.005638	0.046235	0.989286	00:48
8	0.003291	0.038489	0.992143	00:49
9	0.001797	0.040723	0.992619	00:47
10	0.000823	0.035182	0.991905	00:49
11	0.000425	0.033875	0.991905	00:48

1.0.1 We are getting 99% accuracy on our valid set

1.1 Let’s get our predictions for our validation set

path=TEST
f=os.listdir(TEST)
new=[str(path)+'/'+s for s in f]


test_dl=learn.dls.test_dl(new)

class_score,y=learn.get_preds(dl=test_dl)

class_score

tensor([[7.9710e-09, 1.0000e+00, 5.3882e-09,  ..., 1.5793e-07, 9.2361e-10, 3.0881e-09],
        [1.0000e+00, 2.6406e-11, 1.7783e-10,  ..., 2.6252e-11, 1.7040e-10, 3.4339e-11],
        [1.0057e-08, 3.5678e-10, 1.7419e-08,  ..., 1.3101e-09, 1.0000e+00, 1.4072e-08],
        ...,
        [2.8537e-08, 1.4790e-08, 3.6033e-09,  ..., 1.4994e-08, 3.6144e-08, 4.4814e-07],
        [1.2168e-09, 4.7895e-13, 4.1590e-10,  ..., 1.4900e-11, 1.0000e+00, 3.3751e-09],
        [2.3216e-10, 8.6410e-08, 1.0862e-09,  ..., 7.7182e-09, 5.8203e-09, 1.9867e-07]])

1.2 We get predictions for each label and get the maximum probability.

class_score = np.argmax(class_score, axis=1)

class_score[1].item()

Let’s Submit our data

sample_submission =  pd.read_csv(INPUT/"sample_submission.csv")
display(sample_submission.head(2))
display(sample_submission.tail(2))

	ImageId	Label
0	1	0
1	2	0

	ImageId	Label
27998	27999	0
27999	28000	0

# remove file extension from filename
ImageId = [os.path.splitext(path)[0] for path in os.listdir(TEST)]
# typecast to int so that file can be sorted by ImageId
ImageId = [int(path) for path in ImageId]
# +1 because index starts at 1 in the submission file
ImageId = [ID+1 for ID in ImageId]

submission  = pd.DataFrame({
    "ImageId": ImageId,
    "Label": class_score
})
# submission.sort_values(by=["ImageId"], inplace = True)
submission.to_csv("submission.csv", index=False)
display(submission.head(3))
display(submission.tail(3))

	ImageId	Label
0	21368	1
1	26151	0
2	21961	8

	ImageId	Label
27997	19739	4
27998	8680	8
27999	43	4