Most Commonly Used 3,000 Kanjis in Japanese

## Code to generate MOST COMMONLY USED 3000 KANJIS

# Import needed libraries
import pandas as pd
import json

# Read data from the appropriate sheet
orig_df = pd.read_excel(
 "data/VDRJ_Ver1_1_Research_Top60894.xlsx",
 sheet_name="重要度順語彙リスト60894語",
)

# Function that detects whether a word includes a kanji
def kanji_detector(word):
 kanji = False
 non_jp_char = False
 for char in word:
  if (
   "\u4e00" <= char <= "\u9faf"
   or "\u3400" <= char <= "\u4dbf"
   or "\uf900" <= char <= "\ufaff"
  ):
   kanji = True
  if (
   not ("\u4e00" <= char <= "\u9faf")
   and not ("\u3400" <= char <= "\u4dbf")
   and not ("\uf900" <= char <= "\ufaff")
   and not ("\u3040" <= char <= "\u30ff")
  ):
   non_jp_char = True

 return kanji and not (non_jp_char)

# Get the relevant column
df = (
 orig_df[["見出し語彙素\nLexeme"]]
 .astype(str)
 .rename(columns={"見出し語彙素\nLexeme": "word"})
)

# Get words that are at least 2 character long and includes a kanji
df = df[df["word"].str.len() > 1]
df["kanji"] = df["word"].apply(kanji_detector)
df = df[df["kanji"]].drop("kanji", axis=1).reset_index(drop=True)

# Loop through the data to calculate the prevalance score of each Kanji
kanjis = {}
for index, word in enumerate(df["word"].to_list()):
 for char in word:
  if "\u4e00" <= char <= "\u9faf" or "\u3400" <= char <= "\u4dbf":
   if char in kanjis:
    kanjis[char]["count"] += 1
    # Prevalence score is calculated as the inverse of commonness of a word
    # If a word is common, it will contribute higher to the rank of Kanji
    kanjis[char]["score"] += 1 / (index + 1)
    if word not in [i["word"] for i in kanjis[char]["words"]]:
     ## Importance shows how commonly used that word is, lower the better
     kanjis[char]["words"].append({"word": word, "importance": index})
   else:
    kanjis[char] = {
     "count": 1,
     "score": 1 / (index + 1),
     "words": [{"word": word, "importance": index}],
    }

# Sort kanjis based on their prevalence 
kanjis_list = sorted(
 [{"kanji": i, **kanjis[i]} for i in kanjis], key=lambda x: x["score"], reverse=True
)

# Get kanjis only if they are used in at least 3 words
kanjis_list = [i for i in kanjis_list if len(i["words"]) > 3]

# Remove some ambiguous kanji that are not really commonly used
unwanted_kanjis = [
 "其",
 "御",
 "為",
 "此",
 "鱻",
 "詞",
 "掛",
 "遣",
 "又",
]

kanjis_list = [i for i in kanjis_list if i["kanji"] not in unwanted_kanjis]

# Exclude some more kanjis that include an ambiguous kanji
for i in kanjis_list:
 i["words"] = [j for j in i["words"] if "鱻" not in j["word"]]

# Check a snapshot of results
for index, i in enumerate(kanjis_list[:100]):
 print(f'{index}.{i["kanji"]}', end=" ")

# Save the top 3000 kanjis
with open("danyel_koca_most_common_3000_kanjis.json", "w") as f:
 json.dump(kanjis_list[:3000], f, ensure_ascii=False)