package main
import (
"archive/zip"
"bufio"
"encoding/xml"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"maps"
"net/http"
"os"
"path"
"slices"
"strconv"
"strings"
"unicode/utf8"
)
var (
emojiData = flag.String("emoji-data", "https://www.unicode.org/Public/16.0.0/ucd/emoji/emoji-data.txt", "URL for emoji data file")
emojiSequences = flag.String("emoji-sequences", "https://www.unicode.org/Public/emoji/16.0/emoji-sequences.txt", "URL for emoji sequences file")
emojiZWJSequences = flag.String("emoji-zwj-sequences", "https://www.unicode.org/Public/emoji/16.0/emoji-zwj-sequences.txt", "URL for emoji ZWJ sequences file")
cldr = flag.String("cldr", "https://unicode.org/Public/cldr/46/cldr-common-46.0.zip", "URL for CLDR data")
c = flag.String("c", "", "Cache dir")
)
func getCached(url, cacheDir string) (string, error) {
cachePath := cacheDir + "/" + path.Base(url)
if _, err := os.Stat(cachePath); !errors.Is(err, fs.ErrNotExist) {
return cachePath, nil
}
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("get %q: %s", url, resp.Status)
}
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return "", err
}
f, err := os.Create(cachePath)
if err != nil {
return "", err
}
if _, err := io.Copy(f, resp.Body); err != nil {
f.Close()
os.Remove(cachePath)
return "", fmt.Errorf("write %q: %s", cachePath, err)
}
if err := f.Close(); err != nil {
os.Remove(cachePath)
return "", err
}
return cachePath, nil
}
func parseEmojiDataLine(line string) (emojis []string, tag string, err error) {
line, _, _ = strings.Cut(line, "#")
if line == "" {
return nil, "", nil
}
parts := strings.SplitN(line, ";", 3)
if len(parts) < 2 {
return nil, "", fmt.Errorf("parse line %q: %s", line, err)
}
codepointsStr := strings.TrimSpace(parts[0])
tag = strings.TrimSpace(parts[1])
if startStr, endStr, ok := strings.Cut(codepointsStr, ".."); ok {
start, err := strconv.ParseInt(startStr, 16, 32)
if err != nil {
return nil, "", fmt.Errorf("failed to parse line %q: %s", line, err)
}
end, err := strconv.ParseInt(endStr, 16, 32)
if err != nil {
return nil, "", fmt.Errorf("failed to parse line %q: %s", line, err)
}
emojis := make([]string, 0, end-start+1)
for i := start; i <= end; i++ {
emojis = append(emojis, string([]rune{rune(i)}))
}
return emojis, tag, nil
}
codepoints := strings.Split(codepointsStr, " ")
emoji := make([]rune, len(codepoints))
for i, codepointStr := range codepoints {
codepoint, err := strconv.ParseInt(codepointStr, 16, 32)
if err != nil {
return nil, "", fmt.Errorf("failed to parse line %q: %s", line, err)
}
emoji[i] = rune(codepoint)
}
return []string{string(emoji)}, tag, nil
}
func emojiModifiers(cacheDir string) ([]string, error) {
var modifiers []string
filePath, err := getCached(*emojiData, cacheDir)
if err != nil {
return nil, err
}
f, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
emojis, tag, err := parseEmojiDataLine(line)
if err != nil {
return nil, err
}
if tag == "Emoji_Modifier" {
modifiers = append(modifiers, emojis...)
}
}
return modifiers, scanner.Err()
}
func emojisInFile(url string, modifiers map[string]bool, cacheDir string) ([]string, error) {
file, err := getCached(url, cacheDir)
if err != nil {
return nil, err
}
var emojis []string
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
lineEmojis, _, err := parseEmojiDataLine(line)
if err != nil {
return nil, err
}
for _, e := range lineEmojis {
if !modifiers[e] {
emojis = append(emojis, e)
}
}
}
return emojis, scanner.Err()
}
func emojis(cacheDir string) ([]string, error) {
modifiersSlice, err := emojiModifiers(cacheDir)
if err != nil {
return nil, err
}
modifiers := make(map[string]bool, len(modifiersSlice))
for _, m := range modifiersSlice {
modifiers[m] = true
}
sequences, err := emojisInFile(*emojiSequences, modifiers, cacheDir)
if err != nil {
return nil, err
}
zwjSequences, err := emojisInFile(*emojiZWJSequences, modifiers, cacheDir)
if err != nil {
return nil, err
}
return append(sequences, zwjSequences...), nil
}
func annotationsInFile(f io.Reader) (map[string]string, error) {
contents, err := io.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("read CLDR data: %s", err)
}
var annotations struct {
Annotations []struct {
CP string `xml:"cp,attr"`
Type string `xml:"type,attr"`
Annotation string `xml:",chardata"`
} `xml:"annotations>annotation"`
}
if err := xml.Unmarshal(contents, &annotations); err != nil {
return nil, fmt.Errorf("parse CLDR data: %s", err)
}
type annotationData struct {
name string
annotations []string
}
emojiAnnotations := make(map[string]annotationData)
for _, annotation := range annotations.Annotations {
annotationData := emojiAnnotations[annotation.CP]
if annotation.Type == "tts" {
annotationData.name = annotation.Annotation
} else {
annotations := strings.Split(annotation.Annotation, "|")
for i, a := range annotations {
annotations[i] = strings.TrimSpace(a)
}
annotationData.annotations = annotations
}
emojiAnnotations[annotation.CP] = annotationData
}
annotationsCombined := make(map[string]string, len(emojiAnnotations))
for emoji, annotation := range emojiAnnotations {
annotation.annotations = slices.DeleteFunc(annotation.annotations, func(s string) bool { return strings.Contains(annotation.name, s) })
annotationsStr := strings.ReplaceAll(annotation.name, ": ", " ")
annotationsStr = strings.ReplaceAll(annotationsStr, ", ", " ")
if len(annotation.annotations) > 0 {
annotationsStr += " " + strings.Join(annotation.annotations, " ")
}
annotationsCombined[emoji] = annotationsStr
}
return annotationsCombined, nil
}
func annotations(cldrData *zip.Reader) (map[string]string, error) {
annotationsFile, err := cldrData.Open("common/annotations/en.xml")
if err != nil {
return nil, fmt.Errorf("read CLDR data: %s", err)
}
defer annotationsFile.Close()
annotations, err := annotationsInFile(annotationsFile)
if err != nil {
return nil, err
}
annotationsDerivedFile, err := cldrData.Open("common/annotationsDerived/en.xml")
if err != nil {
return nil, fmt.Errorf("read CLDR data: %s", err)
}
defer annotationsDerivedFile.Close()
annotationsDerived, err := annotationsInFile(annotationsDerivedFile)
if err != nil {
return nil, err
}
maps.Copy(annotations, annotationsDerived)
return annotations, err
}
func removePresentationSelector(emoji string) string {
return strings.ReplaceAll(emoji, "\ufe0f", "")
}
func collationData(cldrData *zip.Reader) (func(string) int, error) {
file, err := cldrData.Open("common/collation/root.xml")
if err != nil {
return nil, fmt.Errorf("read CLDR data: %s", err)
}
defer file.Close()
content, err := io.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("read CLDR data: %s", err)
}
var collations struct {
Collation []struct {
Type string `xml:"type,attr"`
CR string `xml:"cr"`
} `xml:"collations>collation"`
}
if err := xml.Unmarshal(content, &collations); err != nil {
return nil, fmt.Errorf("parse CLDR data %q: %s", file, err)
}
var emojiCollation string
for _, collation := range collations.Collation {
if collation.Type == "emoji" {
emojiCollation = collation.CR
break
}
}
if emojiCollation == "" {
return nil, fmt.Errorf("no emoji collation found in %q", file)
}
// 🫤
ignorable := make(map[rune]bool)
collation := make(map[string]int)
count := 1
for _, line := range strings.Split(emojiCollation, "\n") {
if line, ok := strings.CutPrefix(line, "& [last primary ignorable]<<*"); ok {
for _, rune := range line {
ignorable[rune] = true
}
}
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "&") {
continue
}
if strings.HasPrefix(line, "<*") {
for _, emoji := range line[2:] {
collation[string([]rune{emoji})] = count
count++
}
} else if strings.HasPrefix(line, "<") {
for _, emoji := range strings.FieldsFunc(line[1:], func(r rune) bool { return r == ' ' || r == '<' || r == '=' || r == '\'' }) {
collation[emoji] = count
count++
}
} else {
return nil, fmt.Errorf("unexpected line format %q", line)
}
}
return func(emoji string) int {
// First try the minimally qualified version, with one trailing
// modifier removed.
minimallyQualified := removePresentationSelector(emoji)
lastRune, size := utf8.DecodeLastRuneInString(minimallyQualified)
if size < len(minimallyQualified) && ignorable[lastRune] {
minimallyQualified = minimallyQualified[:len(minimallyQualified)-size]
minimallyQualified = strings.TrimSuffix(minimallyQualified, "\u200d") // ZWJ
}
if n, ok := collation[minimallyQualified]; ok {
return n
}
// If that's not found, try the fully unqualified version,
// where we remove all modifiers.
unqualified := minimallyQualified
var unqualifiedRunes []rune
for _, r := range unqualified {
if !ignorable[r] {
unqualifiedRunes = append(unqualifiedRunes, r)
}
}
unqualified = string(unqualifiedRunes)
unqualified = strings.TrimRight(unqualified, "\u200d") // ZWJ
if n, ok := collation[unqualified]; ok {
return n
}
// Finally, fall back to checking the first codepoint.
firstRune, _ := utf8.DecodeRuneInString(emoji)
if n, ok := collation[string([]rune{firstRune})]; ok {
return n
}
fmt.Fprintf(os.Stderr, "Unable to classify emoji %s", emoji)
os.Exit(1)
panic("unreachable")
}, nil
}
func generate() error {
cacheDir := *c
if cacheDir == "" {
var err error
cacheDir, err = os.MkdirTemp("", "")
if err != nil {
return err
}
defer os.RemoveAll(cacheDir)
}
emojis, err := emojis(cacheDir)
if err != nil {
return err
}
cldrFile, err := getCached(*cldr, cacheDir)
if err != nil {
return err
}
cldrData, err := zip.OpenReader(cldrFile)
if err != nil {
return err
}
defer cldrData.Close()
annotations, err := annotations(&cldrData.Reader)
if err != nil {
return err
}
collate, err := collationData(&cldrData.Reader)
if err != nil {
return err
}
slices.SortFunc(emojis, func(e, f string) int {
if n := collate(e) - collate(f); n != 0 {
return n
}
if n := strings.Count(e, "\u200d") - strings.Count(f, "\u200d"); n != 0 {
return n
}
return slices.Compare([]rune(e), []rune(f))
})
for _, emoji := range emojis {
// From CLDR: "Warnings: All cp values have U+FE0F characters removed."
// So we have to remove all fe0f characters for some reason.
annotation, ok := annotations[removePresentationSelector(emoji)]
if !ok {
return fmt.Errorf("emoji %q has no annotation", emoji)
}
fmt.Println(emoji, annotation)
}
return nil
}
func main() {
flag.Parse()
if err := generate(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
