001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.common;
019
020import java.net.URISyntaxException;
021
022/**
023 * Implementation of an URI profile. A profile can be used to customize which
024 * characters and features are acceptable when a certain profile is used.
025 *
026 * An array of integers is used to defined which categories the first
027 * 8-bit characters belong to.
028 *
029 * @author nicl
030 */
031public class UriProfile {
032
033    /** Bit to categorize a char as an alpha. */
034    public static final int B_ALPHAS = 1 << 0;
035    /** Bit to categorize a char as a digit. */
036    public static final int B_DIGITS = 1 << 1;
037    /** Bit to categorize a char as first in scheme. */
038    public static final int B_SCHEME_FIRST = 1 << 2;
039    /** Bit to categorize a char as following in scheme. */
040    public static final int B_SCHEME_FOLLOW = 1 << 3;
041    /** Bit to categorize a char as UNRESERVED in RFC3986. */
042    public static final int B_UNRESERVED = 1 << 4;
043    /** Bit to categorize a char as GEN-DELIMS in RFC3986. */
044    public static final int B_GEN_DELIMS = 1 << 5;
045    /** Bit to categorize a char as SUB-DELIMS in RFC3986. */
046    public static final int B_SUB_DELIMS = 1 << 6;
047    /** Bit to categorize a char as RESERVED in RFC3986. */
048    public static final int B_RESERVED = 1 << 7;
049    /** Bit to categorize a char as PCHAR in RFC3986. */
050    public static final int B_PCHAR = 1 << 8;
051    /** Bit to categorize a char as USERINFO in RFC3986. */
052    public static final int B_USERINFO = 1 << 9;
053    /** Bit to categorize a char as REGNAME in RFC3986. */
054    public static final int B_REGNAME = 1 << 10;
055    /** Bit to categorize a char as SEGMENT in RFC3986. */
056    public static final int B_SEGMENT = 1 << 11;
057    /** Bit to categorize a char as SEGMENT-NZ in RFC3986. */
058    public static final int B_SEGMENT_NZ = 1 << 12;
059    /** Bit to categorize a char as SEGMENT-NZ-NC in RFC3986. */
060    public static final int B_SEGMENT_NZ_NC = 1 << 13;
061    /** Bit to categorize a char as PATH in RFC3986. */
062    public static final int B_PATH = 1 << 14;
063    /** Bit to categorize a char as QUERY in RFC3986. */
064    public static final int B_QUERY = 1 << 15;
065    /** Bit to categorize a char as FRAGMENT in RFC3986. */
066    public static final int B_FRAGMENT = 1 << 16;
067
068    /** Array of integers used to categorize all 8bit chars. */
069    protected final int[] charTypeMap = new int[256];
070
071    /** Does profile allow relative URIs. */
072    public boolean bAllowRelativeUris;
073
074    /** Does profile allow 16-bit percent encoding. */
075    public boolean bAllow16bitPercentEncoding;
076
077    /** Does profile allow invalid percent encoding. */
078    public boolean bAllowInvalidPercentEncoding;
079
080    /**
081     * Construct an <code>UriProfile</code> initialized with RFC3986
082     * rules.
083     */
084    public UriProfile() {
085        for (int i=0; i<defaultCharTypeMap.length; ++i) {
086            charTypeMap[i] = defaultCharTypeMap[i];
087        }
088        bAllowRelativeUris = true;
089        bAllow16bitPercentEncoding = false;
090        bAllowInvalidPercentEncoding = false;
091    }
092
093    /**
094     * Construct an <code>UriProfile</code> initialized from another profile.
095     * @param uriProfile URI profile to base a new profile on
096     */
097    public UriProfile(UriProfile uriProfile) {
098        for (int i=0; i<charTypeMap.length; ++i) {
099            charTypeMap[i] = uriProfile.charTypeMap[i];
100        }
101        bAllowRelativeUris = uriProfile.bAllowRelativeUris;
102        bAllow16bitPercentEncoding = uriProfile.bAllow16bitPercentEncoding;
103        bAllowInvalidPercentEncoding = uriProfile.bAllowInvalidPercentEncoding;
104    }
105
106    /**
107     * Helper method to populate the profiles integer array with character
108     * categories.
109     * @param chars characters to categorize or null
110     * @param bw_and character categories to include in new category
111     * @param bw_or new category/categories
112     */
113    public void charTypeAddAndOr(String chars, int bw_and, int bw_or) {
114        UriProfile.charTypeAddAndOr(charTypeMap, chars, bw_and, bw_or);
115    }
116
117    /**
118     * Given a collection of characters find the index of the first occurrence
119     * of any of the characters in the supplied string starting from a certain
120     * position.
121     * @param bw_and bits identifying one or more character categories
122     * @param str string to search through
123     * @param pos position to start searching from
124     * @return index of the first character in the string present in the
125     * character collection(s)
126     * @throws URISyntaxException if a character with a value larger that 255
127     * is encountered
128     */
129    public int indexOf(int bw_and, String str, int pos) throws URISyntaxException {
130        int limit = str.length();
131        char c;
132        while (pos < limit) {
133            c = str.charAt(pos);
134            if (c < 256) {
135                if ((charTypeMap[c] & bw_and) != 0) {
136                    return pos;
137                }
138                ++pos;
139            } else {
140                throw new URISyntaxException(str, "Invalid URI character '" + (Character.isISOControl(c)?String.format("0x%02x", (int)c):c) + "'");
141            }
142        }
143        return -1;
144    }
145
146    /**
147     * Validate an URI component using two character categories. One category
148     * for the first character and another for the following characters.
149     * @param str URI component string
150     * @param bw_and_first bits identifying first character categories
151     * @param bw_and_follow bits identifying following character categories
152     * @throws URISyntaxException if an error occurs parsing component
153     */
154    public void validate_first_follow(String str, int bw_and_first, int bw_and_follow) throws URISyntaxException {
155        int pos = 0;
156        int limit = str.length();
157        char c;
158        while (pos < limit) {
159            c = str.charAt(pos);
160            // indexOf has ensured that the scheme does not have character values > 255.
161            if (pos == 0 && ((charTypeMap[c] & UriProfile.B_SCHEME_FIRST) == 0)) {
162                throw new URISyntaxException(str, "Invalid URI scheme component");
163            } else if ((charTypeMap[c] & UriProfile.B_SCHEME_FOLLOW) == 0) {
164                throw new URISyntaxException(str, "Invalid URI scheme component");
165            }
166            ++pos;
167        }
168    }
169
170    /**
171     * Validates an URI component according to the supplied character category
172     * bitfield.
173     * @param bw_and bits identifying one or more character categories
174     * @param componentName URI component name
175     * @param str URI component string
176     * @return decoded and validated string
177     * @throws URISyntaxException if an error occurs parsing/validating component
178     */
179    public String validate_decode(int bw_and, String componentName, String str) throws URISyntaxException {
180        StringBuilder sb = new StringBuilder();
181        int pos = 0;
182        int ppos;
183        int limit = str.length();
184        char c;
185        int decode = 0;
186        int tmpC;
187        char decodedC;
188        boolean bValid;
189        while (pos < limit) {
190            c = str.charAt(pos++);
191            if (c < 256) {
192                if ((charTypeMap[c] & bw_and) == 0) {
193                    if (c == '%') {
194                        ppos = pos - 1;
195                        if (pos < limit) {
196                            c = str.charAt(pos);
197                            if (c == 'u' || c == 'U') {
198                                if (!bAllow16bitPercentEncoding) {
199                                    if (!bAllowInvalidPercentEncoding) {
200                                        throw new URISyntaxException(str, "Invalid URI " + componentName + " component - 16-bit percent encoding not allowed");
201                                    } else {
202                                        bValid = false;
203                                    }
204                                } else {
205                                    ++pos;
206                                    decode = 4;
207                                    bValid = true;
208                                }
209                            } else {
210                                decode = 2;
211                                bValid = true;
212                            }
213                            decodedC = 0;
214                            while (bValid && decode > 0) {
215                                if (pos < limit) {
216                                    c = str.charAt(pos++);
217                                    decodedC <<= 4;
218                                    if (c < 256) {
219                                        tmpC = asciiHexTab[c];
220                                        if (tmpC != -1) {
221                                            decodedC |= tmpC;
222                                            --decode;
223                                        } else {
224                                            bValid = false;
225                                        }
226                                    } else {
227                                        bValid = false;
228                                    }
229                                } else {
230                                    bValid = false;
231                                }
232                            }
233                            if (!bValid && !bAllowInvalidPercentEncoding) {
234                                throw new URISyntaxException(str, "Invalid URI " + componentName + " component - invalid percent encoding");
235                            }
236                            sb.append((char) decodedC);
237                        } else {
238                            if (!bAllowInvalidPercentEncoding) {
239                                throw new URISyntaxException(str, "Invalid URI " + componentName + " component - incomplete percent encoding");
240                            } else {
241                                bValid = false;
242                            }
243                        }
244                        if (!bValid) {
245                            while (ppos < pos) {
246                                sb.append(str.charAt(ppos++));
247                            }
248                        }
249                    } else {
250                        throw new URISyntaxException(str, "Invalid URI " + componentName + " component - invalid character '" + (Character.isISOControl(c)?String.format("0x%02x", (int)c):c) + "'");
251                    }
252                } else {
253                    sb.append(c);
254                }
255            } else {
256                throw new URISyntaxException(str, "Invalid URI " + componentName + " component - invalid character '" + (Character.isISOControl(c)?String.format("0x%02x", (int)c):c) + "'");
257            }
258        }
259        return sb.toString();
260    }
261
262    /** Hex char to integer conversion table. */
263    public static int[] asciiHexTab = new int[256];
264
265    /** Integer to hex char conversion table. */
266    public static char[] hexTab = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
267
268    /*
269     * Initialize ASCII hex table.
270     */
271    static {
272        String hex = "0123456789abcdef";
273        for (int i=0; i<asciiHexTab.length; ++i) {
274            asciiHexTab[i] = hex.indexOf(i);
275        }
276        hex = hex.toUpperCase();
277        for (int i=0; i<hex.length(); ++i) {
278            asciiHexTab[hex.charAt(i)] = i;
279        }
280    }
281
282    /**
283     * Helper method to populate an integer array with character categories.
284     * @param charTypeMap integer array with character categories
285     * @param chars characters to categorize or null
286     * @param bw_and character categories to include in new category
287     * @param bw_or new category/categories
288     */
289    public static void charTypeAddAndOr(int[] charTypeMap, String chars, int bw_and, int bw_or) {
290        if (chars != null) {
291            for (int i=0; i<chars.length(); ++i) {
292                charTypeMap[chars.charAt(i)] |= bw_or;
293            }
294        }
295        if (bw_and != 0) {
296            for (int i=0; i<charTypeMap.length; ++i) {
297                if ((charTypeMap[i] & bw_and) != 0) {
298                    charTypeMap[i] |= bw_or;
299                }
300            }
301        }
302    }
303
304    /** Default array of integers used to categorize all 8bit chars. */
305    protected static int[] defaultCharTypeMap = new int[256];
306
307    /*
308     * Initialize default char type map.
309     */
310    static {
311        // Alphas.
312        String alphas = "abcdefghijklmnopqrstuvwxyz";
313        charTypeAddAndOr(defaultCharTypeMap, alphas, 0, B_ALPHAS);
314        charTypeAddAndOr(defaultCharTypeMap, alphas.toUpperCase(), 0, B_ALPHAS);
315
316        // Digits.
317        String digits = "1234567890";
318        charTypeAddAndOr(defaultCharTypeMap, digits, 0, B_DIGITS);
319
320        // scheme first/follow.
321        // scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
322        String scheme = "+-.";
323        charTypeAddAndOr(defaultCharTypeMap, null, B_ALPHAS, B_SCHEME_FIRST | B_SCHEME_FOLLOW);
324        charTypeAddAndOr(defaultCharTypeMap, scheme, B_DIGITS, B_SCHEME_FOLLOW);
325
326        // unreserved  = ALPHA / DIGIT / "-" / "." / "_" / "~"
327        String unreserved = "-._~";
328        charTypeAddAndOr(defaultCharTypeMap, unreserved, B_ALPHAS | B_DIGITS, B_UNRESERVED);
329
330        // gen-delims (reserved)
331        String genDelims = ":/?#[]@";
332        charTypeAddAndOr(defaultCharTypeMap, genDelims, 0, B_GEN_DELIMS | B_RESERVED);
333
334        // sub-delims (reserved)
335        String subDelims  = "!$&'()*+,;=";
336        charTypeAddAndOr(defaultCharTypeMap, subDelims, 0, B_SUB_DELIMS | B_RESERVED);
337
338        // pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
339        String pchar = ":@";
340        charTypeAddAndOr(defaultCharTypeMap, pchar, B_UNRESERVED | B_SUB_DELIMS, B_PCHAR);
341
342        // userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
343        String userinfo = ":";
344        charTypeAddAndOr(defaultCharTypeMap, userinfo, B_UNRESERVED | B_SUB_DELIMS, B_USERINFO);
345
346        // reg-name = *( unreserved / pct-encoded / sub-delims )
347        charTypeAddAndOr(defaultCharTypeMap, null, B_UNRESERVED | B_SUB_DELIMS, B_REGNAME);
348
349        // segment = *pchar
350        // segment-nz = 1*pchar
351        charTypeAddAndOr(defaultCharTypeMap, null, B_PCHAR, B_SEGMENT);
352        charTypeAddAndOr(defaultCharTypeMap, null, B_PCHAR, B_SEGMENT_NZ);
353
354        // segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
355        // non-zero-length segment without any colon ":"
356        String segment_nz_nc = "@";
357        charTypeAddAndOr(defaultCharTypeMap, segment_nz_nc, B_UNRESERVED | B_SUB_DELIMS, B_SEGMENT_NZ_NC);
358
359        // path
360        String path = "/";
361        charTypeAddAndOr(defaultCharTypeMap, path, B_PCHAR, B_PATH);
362
363        // query = *( pchar / "/" / "?" )
364        String query =  "/?";
365        charTypeAddAndOr(defaultCharTypeMap, query, B_PCHAR, B_QUERY);
366
367        // fragment = *( pchar / "/" / "?" )
368        String fragment = "/?";
369        charTypeAddAndOr(defaultCharTypeMap, fragment, B_PCHAR, B_FRAGMENT);
370    }
371
372    /** RFC3986 compliant URI profile. */
373    public static final UriProfile RFC3986;
374
375    /** Modified RFC3986 URI profile which disallows relative URIs
376     *  and allows 16-bit percent encoding. */
377    public static final UriProfile RFC3986_ABS_16BIT;
378
379    /** Relaxed RFC3986 URI profile which disallows relative URIs,
380     *  allows 16-bit percent encoding, allow invalid percent encoding
381     *  and allows most 8-bit characters. */
382    public static final UriProfile RFC3986_ABS_16BIT_LAX;
383
384    /*
385     * Initialize default profile.
386     */
387    static {
388        RFC3986 = new UriProfile();
389
390        RFC3986_ABS_16BIT = new UriProfile();
391        RFC3986_ABS_16BIT.bAllowRelativeUris = false;
392        RFC3986_ABS_16BIT.bAllow16bitPercentEncoding = true;
393        RFC3986_ABS_16BIT.bAllowInvalidPercentEncoding = false;
394
395        StringBuilder sb = new StringBuilder("[]");
396        for (int i=33; i<127; ++i) {
397            if ((defaultCharTypeMap[i] & (B_FRAGMENT | B_RESERVED)) == 0 && i != '%') {
398                sb.append((char) i);
399            }
400        }
401        for (int i=161; i<255; ++i) {
402            sb.append((char) i);
403        }
404        // The following characters are accepted in the relaxed mode. And NOT in the official RFC.
405        // []"<>\^`{|}¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþ
406        RFC3986_ABS_16BIT_LAX = new UriProfile();
407        RFC3986_ABS_16BIT_LAX.bAllowRelativeUris = false;
408        RFC3986_ABS_16BIT_LAX.bAllow16bitPercentEncoding = true;
409        RFC3986_ABS_16BIT_LAX.bAllowInvalidPercentEncoding = true;
410        RFC3986_ABS_16BIT_LAX.charTypeAddAndOr(sb.toString(), 0, B_PATH | B_QUERY | B_FRAGMENT);
411        RFC3986_ABS_16BIT_LAX.charTypeAddAndOr("#", 0, B_FRAGMENT);
412    }
413
414}