0
0
mirror of https://github.com/nodejs/node.git synced 2024-11-30 07:27:22 +01:00
nodejs/lib/url.js
Jordan Sissel 358f0ce96b url: add '.' '+' and '-' in url protocol
- Based on BNF from RFC 1738 section 5.
- Added tests to cover svn+ssh and some other examples
2011-11-04 13:36:06 +01:00

612 lines
20 KiB
JavaScript

// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var punycode = require('punycode');
exports.parse = urlParse;
exports.resolve = urlResolve;
exports.resolveObject = urlResolveObject;
exports.format = urlFormat;
// Reference: RFC 3986, RFC 1808, RFC 2396
// define these here so at least they only have to be
// compiled once on the first module load.
var protocolPattern = /^([a-z0-9.+-]+:)/i,
portPattern = /:[0-9]+$/,
// RFC 2396: characters reserved for delimiting URLs.
delims = ['<', '>', '"', '`', ' ', '\r', '\n', '\t'],
// RFC 2396: characters not allowed for various reasons.
unwise = ['{', '}', '|', '\\', '^', '~', '[', ']', '`'].concat(delims),
// Allowed by RFCs, but cause of XSS attacks. Always escape these.
autoEscape = ['\''],
// Characters that are never ever allowed in a hostname.
// Note that any invalid chars are also handled, but these
// are the ones that are *expected* to be seen, so we fast-path
// them.
nonHostChars = ['%', '/', '?', ';', '#']
.concat(unwise).concat(autoEscape),
nonAuthChars = ['/', '@', '?', '#'].concat(delims),
hostnameMaxLen = 255,
hostnamePartPattern = /^[a-zA-Z0-9][a-z0-9A-Z_-]{0,62}$/,
hostnamePartStart = /^([a-zA-Z0-9][a-z0-9A-Z_-]{0,62})(.*)$/,
// protocols that can allow "unsafe" and "unwise" chars.
unsafeProtocol = {
'javascript': true,
'javascript:': true
},
// protocols that never have a hostname.
hostlessProtocol = {
'javascript': true,
'javascript:': true
},
// protocols that always have a path component.
pathedProtocol = {
'http': true,
'https': true,
'ftp': true,
'gopher': true,
'file': true,
'http:': true,
'ftp:': true,
'gopher:': true,
'file:': true
},
// protocols that always contain a // bit.
slashedProtocol = {
'http': true,
'https': true,
'ftp': true,
'gopher': true,
'file': true,
'http:': true,
'https:': true,
'ftp:': true,
'gopher:': true,
'file:': true
},
querystring = require('querystring');
function urlParse(url, parseQueryString, slashesDenoteHost) {
if (url && typeof(url) === 'object' && url.href) return url;
if (typeof url !== 'string') {
throw new TypeError("Parameter 'url' must be a string, not " + typeof url);
}
var out = {},
rest = url;
// cut off any delimiters.
// This is to support parse stuff like "<http://foo.com>"
for (var i = 0, l = rest.length; i < l; i++) {
if (delims.indexOf(rest.charAt(i)) === -1) break;
}
if (i !== 0) rest = rest.substr(i);
var proto = protocolPattern.exec(rest);
if (proto) {
proto = proto[0];
var lowerProto = proto.toLowerCase();
out.protocol = lowerProto;
rest = rest.substr(proto.length);
}
// figure out if it's got a host
// user@server is *always* interpreted as a hostname, and url
// resolution will treat //foo/bar as host=foo,path=bar because that's
// how the browser resolves relative URLs.
if (slashesDenoteHost || proto || rest.match(/^\/\/[^@\/]+@[^@\/]+/)) {
var slashes = rest.substr(0, 2) === '//';
if (slashes && !(proto && hostlessProtocol[proto])) {
rest = rest.substr(2);
out.slashes = true;
}
}
if (!hostlessProtocol[proto] &&
(slashes || (proto && !slashedProtocol[proto]))) {
// there's a hostname.
// the first instance of /, ?, ;, or # ends the host.
// don't enforce full RFC correctness, just be unstupid about it.
// If there is an @ in the hostname, then non-host chars *are* allowed
// to the left of the first @ sign, unless some non-auth character
// comes *before* the @-sign.
// URLs are obnoxious.
var atSign = rest.indexOf('@');
if (atSign !== -1) {
// there *may be* an auth
var hasAuth = true;
for (var i = 0, l = nonAuthChars.length; i < l; i++) {
var index = rest.indexOf(nonAuthChars[i]);
if (index !== -1 && index < atSign) {
// not a valid auth. Something like http://foo.com/bar@baz/
hasAuth = false;
break;
}
}
if (hasAuth) {
// pluck off the auth portion.
out.auth = rest.substr(0, atSign);
rest = rest.substr(atSign + 1);
}
}
var firstNonHost = -1;
for (var i = 0, l = nonHostChars.length; i < l; i++) {
var index = rest.indexOf(nonHostChars[i]);
if (index !== -1 &&
(firstNonHost < 0 || index < firstNonHost)) firstNonHost = index;
}
if (firstNonHost !== -1) {
out.host = rest.substr(0, firstNonHost);
rest = rest.substr(firstNonHost);
} else {
out.host = rest;
rest = '';
}
// pull out port.
var p = parseHost(out.host);
var keys = Object.keys(p);
for (var i = 0, l = keys.length; i < l; i++) {
var key = keys[i];
out[key] = p[key];
}
// we've indicated that there is a hostname,
// so even if it's empty, it has to be present.
out.hostname = out.hostname || '';
// validate a little.
if (out.hostname.length > hostnameMaxLen) {
out.hostname = '';
} else {
var hostparts = out.hostname.split(/\./);
for (var i = 0, l = hostparts.length; i < l; i++) {
var part = hostparts[i];
if (!part) continue;
if (!part.match(hostnamePartPattern)) {
var newpart = '';
for (var j = 0, k = part.length; j < k; j++) {
if (part.charCodeAt(j) > 127) {
// we replace non-ASCII char with a temporary placeholder
// we need this to make sure size of hostname is not
// broken by replacing non-ASCII by nothing
newpart += 'x';
} else {
newpart += part[j];
}
}
// we test again with ASCII char only
if (!newpart.match(hostnamePartPattern)) {
var validParts = hostparts.slice(0, i);
var notHost = hostparts.slice(i + 1);
var bit = part.match(hostnamePartStart);
if (bit) {
validParts.push(bit[1]);
notHost.unshift(bit[2]);
}
if (notHost.length) {
rest = '/' + notHost.join('.') + rest;
}
out.hostname = validParts.join('.');
break;
}
}
}
}
// hostnames are always lower case.
out.hostname = out.hostname.toLowerCase();
// IDNA Support: Returns a puny coded representation of "domain".
// It only converts the part of the domain name that
// has non ASCII characters. I.e. it dosent matter if
// you call it with a domain that already is in ASCII.
var domainArray = out.hostname.split('.');
var newOut = [];
for (var i = 0; i < domainArray.length; ++i) {
var s = domainArray[i];
newOut.push(s.match(/[^A-Za-z0-9_-]/) ?
'xn--' + punycode.encode(s) : s);
}
out.hostname = newOut.join('.');
out.host = (out.hostname || '') +
((out.port) ? ':' + out.port : '');
out.href += out.host;
}
// now rest is set to the post-host stuff.
// chop off any delim chars.
if (!unsafeProtocol[lowerProto]) {
// First, make 100% sure that any "autoEscape" chars get
// escaped, even if encodeURIComponent doesn't think they
// need to be.
for (var i = 0, l = autoEscape.length; i < l; i++) {
var ae = autoEscape[i];
var esc = encodeURIComponent(ae);
if (esc === ae) {
esc = escape(ae);
}
rest = rest.split(ae).join(esc);
}
// Now make sure that delims never appear in a url.
var chop = rest.length;
for (var i = 0, l = delims.length; i < l; i++) {
var c = rest.indexOf(delims[i]);
if (c !== -1) {
chop = Math.min(c, chop);
}
}
rest = rest.substr(0, chop);
}
// chop off from the tail first.
var hash = rest.indexOf('#');
if (hash !== -1) {
// got a fragment string.
out.hash = rest.substr(hash);
rest = rest.slice(0, hash);
}
var qm = rest.indexOf('?');
if (qm !== -1) {
out.search = rest.substr(qm);
out.query = rest.substr(qm + 1);
if (parseQueryString) {
out.query = querystring.parse(out.query);
}
rest = rest.slice(0, qm);
} else if (parseQueryString) {
// no query string, but parseQueryString still requested
out.search = '';
out.query = {};
}
if (rest) out.pathname = rest;
if (slashedProtocol[proto] &&
out.hostname && !out.pathname) {
out.pathname = '/';
}
//to support http.request
if (out.pathname || out.search) {
out.path = (out.pathname ? out.pathname : '') +
(out.search ? out.search : '');
}
// finally, reconstruct the href based on what has been validated.
out.href = urlFormat(out);
return out;
}
// format a parsed object into a url string
function urlFormat(obj) {
// ensure it's an object, and not a string url.
// If it's an obj, this is a no-op.
// this way, you can call url_format() on strings
// to clean up potentially wonky urls.
if (typeof(obj) === 'string') obj = urlParse(obj);
var auth = obj.auth || '';
if (auth) {
auth = auth.split('@').join('%40');
for (var i = 0, l = nonAuthChars.length; i < l; i++) {
var nAC = nonAuthChars[i];
auth = auth.split(nAC).join(encodeURIComponent(nAC));
}
auth += '@';
}
var protocol = obj.protocol || '',
host = (obj.host !== undefined) ? auth + obj.host :
obj.hostname !== undefined ? (
auth + obj.hostname +
(obj.port ? ':' + obj.port : '')
) :
false,
pathname = obj.pathname || '',
query = obj.query &&
((typeof obj.query === 'object' &&
Object.keys(obj.query).length) ?
querystring.stringify(obj.query) :
'') || '',
search = obj.search || (query && ('?' + query)) || '',
hash = obj.hash || '';
if (protocol && protocol.substr(-1) !== ':') protocol += ':';
// only the slashedProtocols get the //. Not mailto:, xmpp:, etc.
// unless they had them to begin with.
if (obj.slashes ||
(!protocol || slashedProtocol[protocol]) && host !== false) {
host = '//' + (host || '');
if (pathname && pathname.charAt(0) !== '/') pathname = '/' + pathname;
} else if (!host) {
host = '';
}
if (hash && hash.charAt(0) !== '#') hash = '#' + hash;
if (search && search.charAt(0) !== '?') search = '?' + search;
return protocol + host + pathname + search + hash;
}
function urlResolve(source, relative) {
return urlFormat(urlResolveObject(source, relative));
}
function urlResolveObject(source, relative) {
if (!source) return relative;
source = urlParse(urlFormat(source), false, true);
relative = urlParse(urlFormat(relative), false, true);
// hash is always overridden, no matter what.
source.hash = relative.hash;
if (relative.href === '') {
source.href = urlFormat(source);
return source;
}
// hrefs like //foo/bar always cut to the protocol.
if (relative.slashes && !relative.protocol) {
relative.protocol = source.protocol;
//urlParse appends trailing / to urls like http://www.example.com
if (slashedProtocol[relative.protocol] &&
relative.hostname && !relative.pathname) {
relative.path = relative.pathname = '/';
}
relative.href = urlFormat(relative);
return relative;
}
if (relative.protocol && relative.protocol !== source.protocol) {
// if it's a known url protocol, then changing
// the protocol does weird things
// first, if it's not file:, then we MUST have a host,
// and if there was a path
// to begin with, then we MUST have a path.
// if it is file:, then the host is dropped,
// because that's known to be hostless.
// anything else is assumed to be absolute.
if (!slashedProtocol[relative.protocol]) {
relative.href = urlFormat(relative);
return relative;
}
source.protocol = relative.protocol;
if (!relative.host && !hostlessProtocol[relative.protocol]) {
var relPath = (relative.pathname || '').split('/');
while (relPath.length && !(relative.host = relPath.shift()));
if (!relative.host) relative.host = '';
if (!relative.hostname) relative.hostname = '';
if (relPath[0] !== '') relPath.unshift('');
if (relPath.length < 2) relPath.unshift('');
relative.pathname = relPath.join('/');
}
source.pathname = relative.pathname;
source.search = relative.search;
source.query = relative.query;
source.host = relative.host || '';
source.auth = relative.auth;
source.hostname = relative.hostname || relative.host;
source.port = relative.port;
//to support http.request
if (source.pathname !== undefined || source.search !== undefined) {
source.path = (source.pathname ? source.pathname : '') +
(source.search ? source.search : '');
}
source.slashes = source.slashes || relative.slashes;
source.href = urlFormat(source);
return source;
}
var isSourceAbs = (source.pathname && source.pathname.charAt(0) === '/'),
isRelAbs = (
relative.host !== undefined ||
relative.pathname && relative.pathname.charAt(0) === '/'
),
mustEndAbs = (isRelAbs || isSourceAbs ||
(source.host && relative.pathname)),
removeAllDots = mustEndAbs,
srcPath = source.pathname && source.pathname.split('/') || [],
relPath = relative.pathname && relative.pathname.split('/') || [],
psychotic = source.protocol &&
!slashedProtocol[source.protocol];
// if the url is a non-slashed url, then relative
// links like ../.. should be able
// to crawl up to the hostname, as well. This is strange.
// source.protocol has already been set by now.
// Later on, put the first path part into the host field.
if (psychotic) {
delete source.hostname;
delete source.port;
if (source.host) {
if (srcPath[0] === '') srcPath[0] = source.host;
else srcPath.unshift(source.host);
}
delete source.host;
if (relative.protocol) {
delete relative.hostname;
delete relative.port;
if (relative.host) {
if (relPath[0] === '') relPath[0] = relative.host;
else relPath.unshift(relative.host);
}
delete relative.host;
}
mustEndAbs = mustEndAbs && (relPath[0] === '' || srcPath[0] === '');
}
if (isRelAbs) {
// it's absolute.
source.host = (relative.host || relative.host === '') ?
relative.host : source.host;
source.hostname = (relative.hostname || relative.hostname === '') ?
relative.hostname : source.hostname;
source.search = relative.search;
source.query = relative.query;
srcPath = relPath;
// fall through to the dot-handling below.
} else if (relPath.length) {
// it's relative
// throw away the existing file, and take the new path instead.
if (!srcPath) srcPath = [];
srcPath.pop();
srcPath = srcPath.concat(relPath);
source.search = relative.search;
source.query = relative.query;
} else if ('search' in relative) {
// just pull out the search.
// like href='?foo'.
// Put this after the other two cases because it simplifies the booleans
if (psychotic) {
source.hostname = source.host = srcPath.shift();
//occationaly the auth can get stuck only in host
//this especialy happens in cases like
//url.resolveObject('mailto:local1@domain1', 'local2@domain2')
var authInHost = source.host && source.host.indexOf('@') > 0 ?
source.host.split('@') : false;
if (authInHost) {
source.auth = authInHost.shift();
source.host = source.hostname = authInHost.shift();
}
}
source.search = relative.search;
source.query = relative.query;
//to support http.request
if (source.pathname !== undefined || source.search !== undefined) {
source.path = (source.pathname ? source.pathname : '') +
(source.search ? source.search : '');
}
source.href = urlFormat(source);
return source;
}
if (!srcPath.length) {
// no path at all. easy.
// we've already handled the other stuff above.
delete source.pathname;
//to support http.request
if (!source.search) {
source.path = '/' + source.search;
} else {
delete source.path;
}
source.href = urlFormat(source);
return source;
}
// if a url ENDs in . or .., then it must get a trailing slash.
// however, if it ends in anything else non-slashy,
// then it must NOT get a trailing slash.
var last = srcPath.slice(-1)[0];
var hasTrailingSlash = (
(source.host || relative.host) && (last === '.' || last === '..') ||
last === '');
// strip single dots, resolve double dots to parent dir
// if the path tries to go above the root, `up` ends up > 0
var up = 0;
for (var i = srcPath.length; i >= 0; i--) {
last = srcPath[i];
if (last == '.') {
srcPath.splice(i, 1);
} else if (last === '..') {
srcPath.splice(i, 1);
up++;
} else if (up) {
srcPath.splice(i, 1);
up--;
}
}
// if the path is allowed to go above the root, restore leading ..s
if (!mustEndAbs && !removeAllDots) {
for (; up--; up) {
srcPath.unshift('..');
}
}
if (mustEndAbs && srcPath[0] !== '' &&
(!srcPath[0] || srcPath[0].charAt(0) !== '/')) {
srcPath.unshift('');
}
if (hasTrailingSlash && (srcPath.join('/').substr(-1) !== '/')) {
srcPath.push('');
}
var isAbsolute = srcPath[0] === '' ||
(srcPath[0] && srcPath[0].charAt(0) === '/');
// put the host back
if (psychotic) {
source.hostname = source.host = isAbsolute ? '' :
srcPath.length ? srcPath.shift() : '';
//occationaly the auth can get stuck only in host
//this especialy happens in cases like
//url.resolveObject('mailto:local1@domain1', 'local2@domain2')
var authInHost = source.host && source.host.indexOf('@') > 0 ?
source.host.split('@') : false;
if (authInHost) {
source.auth = authInHost.shift();
source.host = source.hostname = authInHost.shift();
}
}
mustEndAbs = mustEndAbs || (source.host && srcPath.length);
if (mustEndAbs && !isAbsolute) {
srcPath.unshift('');
}
source.pathname = srcPath.join('/');
//to support request.http
if (source.pathname !== undefined || source.search !== undefined) {
source.path = (source.pathname ? source.pathname : '') +
(source.search ? source.search : '');
}
source.auth = relative.auth || source.auth;
source.slashes = source.slashes || relative.slashes;
source.href = urlFormat(source);
return source;
}
function parseHost(host) {
var out = {};
var port = portPattern.exec(host);
if (port) {
port = port[0];
out.port = port.substr(1);
host = host.substr(0, host.length - port.length);
}
if (host) out.hostname = host;
return out;
}