DXR is a code search and navigation tool aimed at making sense of large projects. It supports full-text and regex searches as well as structural queries.

Header

Mercurial (31ec81b5d7bb)

VCS Links

Line Code
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
/*
 **********************************************************************
 *   Copyright (C) 2005-2012, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 **********************************************************************
 */

#include "unicode/utypes.h"

#if !UCONFIG_NO_CONVERSION

#include "cstring.h"

#include "csr2022.h"
#include "csmatch.h"

U_NAMESPACE_BEGIN

#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])

/**
 * Matching function shared among the 2022 detectors JP, CN and KR
 * Counts up the number of legal and unrecognized escape sequences in
 * the sample of text, and computes a score based on the total number &
 * the proportion that fit the encoding.
 * 
 * 
 * @param text the byte buffer containing text to analyse
 * @param textLen  the size of the text in the byte.
 * @param escapeSequences the byte escape sequences to test for.
 * @return match quality, in the range of 0-100.
 */
int32_t CharsetRecog_2022::match_2022(const uint8_t *text, int32_t textLen, const uint8_t escapeSequences[][5], int32_t escapeSequences_length) const
{
    int32_t i, j;
    int32_t escN;
    int32_t hits   = 0;
    int32_t misses = 0;
    int32_t shifts = 0;
    int32_t quality;

    i = 0;
    while(i < textLen) {
        if(text[i] == 0x1B) {
            escN = 0;
            while(escN < escapeSequences_length) {
                const uint8_t *seq = escapeSequences[escN];
                int32_t seq_length = (int32_t)uprv_strlen((const char *) seq);

                if (textLen-i >= seq_length) {
                    j = 1;
                    while(j < seq_length) {
                        if(seq[j] != text[i+j]) {
                            goto checkEscapes;
                        }

                        j += 1;
                    }

                    hits += 1;
                    i += seq_length-1;
                    goto scanInput;
                }
                // else we ran out of string to compare this time.
checkEscapes:
                escN += 1;
            }

            misses += 1;
        }

        if( text[i]== 0x0e || text[i] == 0x0f){
            shifts += 1;
        }

scanInput:
        i += 1;
    }

    if (hits == 0) {
        return 0;
    }

    //
    // Initial quality is based on relative proportion of recongized vs.
    //   unrecognized escape sequences. 
    //   All good:  quality = 100;
    //   half or less good: quality = 0;
    //   linear inbetween.
    quality = (100*hits - 100*misses) / (hits + misses);

    // Back off quality if there were too few escape sequences seen.
    //   Include shifts in this computation, so that KR does not get penalized
    //   for having only a single Escape sequence, but many shifts.
    if (hits+shifts < 5) {
        quality -= (5-(hits+shifts))*10;
    }

    if (quality < 0) {
        quality = 0;
    }

    return quality;
}


static const uint8_t escapeSequences_2022JP[][5] = {
    {0x1b, 0x24, 0x28, 0x43, 0x00},   // KS X 1001:1992
    {0x1b, 0x24, 0x28, 0x44, 0x00},   // JIS X 212-1990
    {0x1b, 0x24, 0x40, 0x00, 0x00},   // JIS C 6226-1978
    {0x1b, 0x24, 0x41, 0x00, 0x00},   // GB 2312-80
    {0x1b, 0x24, 0x42, 0x00, 0x00},   // JIS X 208-1983
    {0x1b, 0x26, 0x40, 0x00, 0x00},   // JIS X 208 1990, 1997
    {0x1b, 0x28, 0x42, 0x00, 0x00},   // ASCII
    {0x1b, 0x28, 0x48, 0x00, 0x00},   // JIS-Roman
    {0x1b, 0x28, 0x49, 0x00, 0x00},   // Half-width katakana
    {0x1b, 0x28, 0x4a, 0x00, 0x00},   // JIS-Roman
    {0x1b, 0x2e, 0x41, 0x00, 0x00},   // ISO 8859-1
    {0x1b, 0x2e, 0x46, 0x00, 0x00}    // ISO 8859-7
};

static const uint8_t escapeSequences_2022KR[][5] = {
    {0x1b, 0x24, 0x29, 0x43, 0x00}   
};

static const uint8_t escapeSequences_2022CN[][5] = {
    {0x1b, 0x24, 0x29, 0x41, 0x00},   // GB 2312-80
    {0x1b, 0x24, 0x29, 0x47, 0x00},   // CNS 11643-1992 Plane 1
    {0x1b, 0x24, 0x2A, 0x48, 0x00},   // CNS 11643-1992 Plane 2
    {0x1b, 0x24, 0x29, 0x45, 0x00},   // ISO-IR-165
    {0x1b, 0x24, 0x2B, 0x49, 0x00},   // CNS 11643-1992 Plane 3
    {0x1b, 0x24, 0x2B, 0x4A, 0x00},   // CNS 11643-1992 Plane 4
    {0x1b, 0x24, 0x2B, 0x4B, 0x00},   // CNS 11643-1992 Plane 5
    {0x1b, 0x24, 0x2B, 0x4C, 0x00},   // CNS 11643-1992 Plane 6
    {0x1b, 0x24, 0x2B, 0x4D, 0x00},   // CNS 11643-1992 Plane 7
    {0x1b, 0x4e, 0x00, 0x00, 0x00},   // SS2
    {0x1b, 0x4f, 0x00, 0x00, 0x00},   // SS3
};

CharsetRecog_2022JP::~CharsetRecog_2022JP() {}

const char *CharsetRecog_2022JP::getName() const {
    return "ISO-2022-JP";
}

UBool CharsetRecog_2022JP::match(InputText *textIn, CharsetMatch *results) const {
    int32_t confidence = match_2022(textIn->fInputBytes, 
                                    textIn->fInputLen, 
                                    escapeSequences_2022JP, 
                                    ARRAY_SIZE(escapeSequences_2022JP));
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_2022KR::~CharsetRecog_2022KR() {}

const char *CharsetRecog_2022KR::getName() const {
    return "ISO-2022-KR";
}

UBool CharsetRecog_2022KR::match(InputText *textIn, CharsetMatch *results) const {
    int32_t confidence = match_2022(textIn->fInputBytes, 
                                    textIn->fInputLen, 
                                    escapeSequences_2022KR, 
                                    ARRAY_SIZE(escapeSequences_2022KR));
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_2022CN::~CharsetRecog_2022CN() {}

const char *CharsetRecog_2022CN::getName() const {
    return "ISO-2022-CN";
}

UBool CharsetRecog_2022CN::match(InputText *textIn, CharsetMatch *results) const {
    int32_t confidence = match_2022(textIn->fInputBytes,
                                    textIn->fInputLen,
                                    escapeSequences_2022CN,
                                    ARRAY_SIZE(escapeSequences_2022CN));
    results->set(textIn, this, confidence);
    return (confidence > 0);
}

CharsetRecog_2022::~CharsetRecog_2022() {
    // nothing to do
}

U_NAMESPACE_END
#endif