>>> import pandas as pd
>>> model_df = pd.read_json('data/mag_papers_0/mag_subset20K.txt', lines=True)
>>> model_df.shape
(20000, 19)
>>> model_df.columns
Index(['abstract', 'authors', 'doc_type', 'doi', 'fos', 'id', 'issue',
'keywords', 'lang', 'n_citation', 'page_end', 'page_start', 'publisher',
'references', 'title', 'url', 'venue', 'volume', 'year'],
dtype='object')
# filter out non-English articles and focus on a few variables
>>> model_df = model_df[model_df.lang == 'en']
... .drop_duplicates(subset='title', keep='first')
... .drop(['doc_type', 'doi', 'id', 'issue', 'lang', 'n_citation',
... 'page_end', 'page_start', 'publisher', 'references',
... 'url', 'venue', 'volume'], axis=1)
>>> model_df.shape
(10399, 6)

>>> unique_fos = sorted(list({feature
... for paper_row in model_df.fos.fillna('0')
... for feature in paper_row}))
>>> unique_year = sorted(model_df['year'].astype('str').unique())
>>> def feature_array(x, var, unique_array):
...     row_dict = {}
...     for i in x.index:
...         var_dict = {}
...         for j in range(len(unique_array)):
...             if type(x[i]) is list:
...                 if unique_array[j] in x[i]:
...                     var_dict.update({var + '_' + unique_array[j]: 1})
...                 else:
...                     var_dict.update({var + '_' + unique_array[j]: 0})
...             else:
...                 if unique_array[j] == str(x[i]):
...                     var_dict.update({var + '_' + unique_array[j]: 1})
...                 else:
...                     var_dict.update({var + '_' + unique_array[j]: 0})
...         row_dict.update({i: var_dict})
...     feature_df = pd.DataFrame.from_dict(row_dict, dtype='str').T
...     return feature_df
>>> year_features = feature_array(model_df['year'], unique_year)
>>> fos_features = feature_array(model_df['fos'], unique_fos)
>>> first_features = fos_features.join(year_features).T
>>> from sys import getsizeof
>>> print('Size of first feature array: ', getsizeof(first_features))
Size of first feature array: 2583077234

>>> from scipy.spatial.distance import cosine
>>> def item_collab_filter(features_df):
...     item_similarities = pd.DataFrame(index=features_df.columns,
...                                      columns=features_df.columns)
...     for i in features_df.columns:
...         for j in features_df.columns:
...             item_similarities.loc[i][j] = 1 - cosine(features_df[i],
...                                                      features_df[j])
...     return item_similarities
>>> first_items = item_collab_filter(first_features.loc[:, 0:1000])

>>> import matplotlib.pyplot as plt
>>> import seaborn as sns
>>> import numpy as np
>>> %matplotlib inline
>>> sns.set()
>>> ax = sns.heatmap(first_items.fillna(0),
...                 vmin=0, vmax=1,
...                 cmap="YlGnBu",
...                 xticklabels=250, yticklabels=250)
>>> ax.tick_params(labelsize=12)

>>> def paper_recommender(paper_ix, items_df):
...     print('Based on the paper: \n index = ', paper_ix)
...     print(model_df.iloc[paper_ix])
...     top_results = items_df.loc[paper_ix].sort_values(ascending=False).head(4)
...     print(' \n Top three results: ')
...     order = 1
...     for i in top_results.index.tolist()[-3:]:
...         print(order, '. Paper index = ', i)
...         print('Similarity score: ', top_results[i])
...         print(model_df.iloc[i], ' \n ')
...         if order < 5: order += 1
>>> paper_recommender(2, first_items)
Based on the paper:
index =  2
abstract                                                  NaN
authors     [{'name': 'Jovana P. Lekovich', 'org': 'Weill ...
fos                                                       NaN
keywords                                                  NaN
title       Should endometriosis be an indication for intr...
year                                                     2015
Name: 2, dtype: object
Top three results:
1 . Paper index =  2
Similarity score:  1.0
abstract                                                  NaN
authors     [{'name': 'Jovana P. Lekovich', 'org': 'Weill ...
fos                                                       NaN
keywords                                                  NaN
title       Should endometriosis be an indication for intr...
year                                                     2015
Name: 2, dtype: object
2 . Paper index =  292
Similarity score:  1.0
abstract                                                  NaN
authors     [{'name': 'John C. Newton'}, {'name': 'Beers M...
fos         [Wide area multilateration, Maneuvering speed,...
keywords                                                  NaN
title                    Automatic speed control for aircraft
year                                                     1955
Name: 561, dtype: object
3 . Paper index =  593
Similarity score:  1.0
abstract    This paper demonstrates that on-site greywater...
authors     [{'name': 'Eran Friedler', 'org': 'Division of...
fos         [Public opinion, Environmental Engineering, Wa...
keywords    [economic analysis, tratamiento desperdicios, ...
title       The water saving potential and the socio-econo...
year                                                     2008
Name: 1152, dtype: object

>>> print("Year spread: ", model_df['year'].min(), " - ", model_df['year'].max())
>>> print("Quantile spread: \n ", model_df['year'].quantile([0.25, 0.5, 0.75]))
Year spread:  1831  -  2017
Quantile spread:
0.25    1990.0
0.50    2005.0
0.75    2012.0
Name: year, dtype: float64
# plot years to see the distribution
>>> fig, ax = plt.subplots()
>>> model_df['year'].hist(ax=ax,
... bins=model_df['year'].max() - model_df['year'].min())
>>> ax.tick_params(labelsize=12)
>>> ax.set_xlabel('Year Count', fontsize=12)
>>> ax.set_ylabel('Occurrence', fontsize=12)

# binning here (by 10 years) reduces the year feature space from 156 to 19
>>> bins = int(round((model_df['year'].max() - model_df['year'].min()) / 10))
>>> temp_df = pd.DataFrame(index=model_df.index)
>>> temp_df['yearBinned'] = pd.cut(model_df['year'].tolist(), bins, precision=0)
>>> X_yrs = pd.get_dummies(temp_df['yearBinned'])
>>> X_yrs.columns.categories
IntervalIndex([(1831.0, 1841.0], (1841.0, 1851.0], (1851.0, 1860.0],
(1860.0, 1870.0], (1870.0, 1880.0] ... (1968.0, 1978.0],
(1978.0, 1988.0], (1988.0, 1997.0], (1997.0, 2007.0],
(2007.0, 2017.0]]
closed='right',
dtype='interval[float64]')
# plot the new distribution
>>> fig, ax = plt.subplots()
>>> X_yrs.sum().plot.bar(ax=ax)
>>> ax.tick_params(labelsize=8)
>>> ax.set_xlabel('Binned Years', fontsize=12)
>>> ax.set_ylabel('Counts', fontsize=12)

>>> X_fos = fos_features.values
# We can see how this will make a difference in the future by looking
# at the size of each
>>> print('Our pandas Series, in bytes: ', getsizeof(fos_features))
>>> print('Our hashed numpy array, in bytes: ', getsizeof(X_fos))
Our pandas Series, in bytes:  2530632380
Our hashed numpy array, in bytes:  112

>>> second_features = np.append(X_fos, X_yrs, axis=1)
>>> print("The power of feature engineering saves us, in bytes: ",
... getsizeof(first_features) - getsizeof(second_features))
The power of feature engineering saves us, in bytes:  168066769
>>> from sklearn.metrics.pairwise import cosine_similarity
>>> def piped_collab_filter(features_matrix, index, top_n):
... item_similarities =
\
... 1 - cosine_similarity(features_matrix[index:index + 1],
... features_matrix).flatten()
... related_indices =
\
... [i for i in item_similarities.argsort()[::-1] if i != index]
... return [(index, item_similarities[index])
... for index in related_indices
... ][0:top_n]

>>> def paper_recommender(items_df, paper_ix, top_n):
... if paper_ix in model_df.index:
... print('Based on the paper:')
... print('Paper index = ', model_df.loc[paper_ix].name)
... print('Title :', model_df.loc[paper_ix]['title'])
... print('FOS :', model_df.loc[paper_ix]['fos'])
... print('Year :', model_df.loc[paper_ix]['year'])
... print('Abstract :', model_df.loc[paper_ix]['abstract'])
... print('Authors :', model_df.loc[paper_ix]['authors'], ' \n ')
... # define the location index for the DataFrame index requested
... array_ix = model_df.index.get_loc(paper_ix)
... top_results = piped_collab_filter(items_df, array_ix, top_n)
... print(' \n Top', top_n, 'results: ')
... order = 1
... for i in range(len(top_results)):
... print(order, '. Paper index = ',
... model_df.iloc[top_results[i][0]].name)
... print('Similarity score: ', top_results[i][1])
... print('Title :', model_df.iloc[top_results[i][0]]['title'])
... print('FOS :', model_df.iloc[top_results[i][0]]['fos'])
... print('Year :', model_df.iloc[top_results[i][0]]['year'])
... print('Abstract :', model_df.iloc[top_results[i][0]]['abstract'])
... print('Authors :', model_df.iloc[top_results[i][0]]['authors'],
... ' \n ')
... if order < top_n: order += 1
... else:
... print('Whoops! Choose another paper. Try something from here: \n ',
... model_df.index[100:200])
>>> paper_recommender(second_features, 2, 3)
Based on the paper:
Paper index =  2
Title : Should endometriosis be an indication for intracytoplasmic sperm inject ...
FOS : nan
Year : 2015
Abstract : nan
Authors : [{'name': 'Jovana P. Lekovich', 'org': 'Weill Cornell Medical College, ...
Top 3 results:
1 . Paper index =  10055
Similarity score:  1.0
Title : [Diagnosis of cerebral tumors; comparative studies on arteriography, ...
FOS : ['Radiology', 'Pathology', 'Surgery']
Year : 1953
Abstract : nan
Authors : [{'name': 'Antoine'}, {'name': 'Lepoire'}, {'name': 'Schoumacker'}]
2 . Paper index =  11771
Similarity score:  1.0
Title : A Study of Special Functions in the Theory of Eclipsing Binary Systems
FOS : ['Contact binary']
Year : 1981
Abstract : nan
Authors : [{'name': 'Filaretti Zafiropoulos', 'org': 'University of Manchester'}]
3 . Paper index =  11773
Similarity score:  1.0
Title : Studies of powder flow using a recording powder flowmeter and measure ...
FOS : nan
Year : 1985
Abstract : This paper describes the utility of the dynamic measurement of the ...
Authors : [{'name': 'Ramachandra P. Hegde', 'org': 'Department of Pharmacy, ...

>>> model_df.loc[21]
abstract    A microprocessor includes hardware registers t...
authors                      [{'name': 'Mark John Ebersole'}]
fos         [Embedded system, Parallel computing, Computer...
keywords                                                  NaN
title       Microprocessor that enables ARM ISA program to...
year                                                     2013
Name: 21, dtype: object
>>> model_df.iloc[21]
abstract                                                  NaN
authors     [{'name': 'Nicola M. Heller'}, {'name': 'Steph...
fos         [Biology, Medicine, Post-transcriptional regul...
keywords    [glucocorticoids, post transcriptional regulat...
title       Post-transcriptional regulation of eotaxin by ...
year                                                     2002
Name: 30, dtype: object
>>> model_df.index.get_loc(30)
21

# need to fill in NaN for sklearn use in future
>>> filled_df = model_df.fillna('None')
>>> from sklearn.feature_extraction.text import TfidfVectorizer
>>> vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
... stop_words='english')
>>> X_abstract = vectorizer.fit_transform(filled_df['abstract'])
>>> third_features = np.append(second_features, X_abstract.toarray(), axis=1)

>>> authors_list = []
>>> for row in filled_df.authors.itertuples():
... # create a dictionary from each Series index
... if type(row.authors) is str:
... y = {'None': row.Index}
... if type(row.authors) is list:
... # add these keys + values to our running dictionary
... y = dict.fromkeys(row.authors[0].values(), row.Index)
... authors_list.append(y)
>>> authors_list[0:5]
[{'None': 0},
{'Ahmed M. Alluwaimi': 1},
{'Jovana P. Lekovich': 2, 'Weill Cornell Medical College, New York, NY': 2},
{'George C. Sponsler': 5},
{'M. T. Richards': 7}]
>>> from sklearn.feature_extraction import DictVectorizer
>>> v = DictVectorizer(sparse=False)
>>> D = authors_list
>>> X_authors = v.fit_transform(D)
>>> fourth_features = np.append(third_features, X_authors, axis=1)

>>> paper_recommender(fourth_features, 2, 3)
Based on the paper:
Paper index =  2
Title : Should endometriosis be an indication for intracytoplasmic sperm inject ...
FOS : nan
Year : 2015
Abstract : nan
Authors : [{'name': 'Jovana P. Lekovich', 'org': 'Weill Cornell Medical College, ...
Top 3 results:
1 . Paper index =  10055
Similarity score:  1.0
Title : [Diagnosis of cerebral tumors; comparative studies on arteriography, ...
FOS : ['Radiology', 'Pathology', 'Surgery']
Year : 1953
Abstract : nan
Authors : [{'name': 'Antoine'}, {'name': 'Lepoire'}, {'name': 'Schoumacker'}]
2 . Paper index =  5601
Similarity score:  1.0
Title : 633 Survival after coronary revascularization, with and without mitral ...
FOS : ['Cardiology']
Year : 2005
Abstract : nan
Authors : [{'name': 'J.B. Le Polain De Waroux'}, {'name': 'Anne-Catherine ...
3 . Paper index =  12256
Similarity score:  1.0
Title : Nucleotide Sequence and Analysis of an Insertion Sequence from Bacillus ...
FOS : ['Biology', 'Molecular biology', 'Insertion sequence', 'Nucleic acid ...
Year : 1994
Abstract : A 5.8-kb DNA fragment encoding the  cryIC  gene from  Bacillus thur...
Authors : [{'name': 'Geoffrey P. Smith'}, {'name': 'David J. Ellar'}, {'name': ...