[Laszlo-dev] [Laszlo-checkins] r7132 - in openlaszlo/trunk/WEB-INF: lib lps/lfc/kernel/swf lps/server/src/org/openlaszlo/data
Max Carlson
max at openlaszlo.org
Wed Nov 7 15:40:31 PST 2007
I think you forgot to check in your updated testcases...
hqm at openlaszlo.org wrote:
> Author: hqm
> Date: 2007-11-05 12:00:28 -0800 (Mon, 05 Nov 2007)
> New Revision: 7132
>
> Added:
> openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
> openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
> Removed:
> openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar
> Modified:
> openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as
> openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
> openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
> openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
> Log:
> Change 20071105-hqm-0 by hqm at IBM-2E06404CB67 on 2007-11-05 14:54:40 EST
> in /cygdrive/c/users/hqm/openlaszlo/trunk
> for http://svn.openlaszlo.org/openlaszlo/trunk
>
> Summary: put XML parser back into server data proxy pipeline, to do
> charset transcoding
>
> New Features:
>
> Bugs Fixed: LPP-4924
>
> Technical Reviewer: max
> QA Reviewer: pablo
> Doc Reviewer:
>
> Details:
>
> The server data proxy now uses the XMLPULL parser to parse
> the data from the backend, in order to use Java to force a translation
> into UTF-8
> coding if needed.
>
> This change also uses a worker Thread to read from the backend, while
> simultaneously
> pipelining the data back throug the XML PULL parser to the client.
> This should improve
> response time and also removes a potential memory overflow and DOS
> attack on the server.
>
> Tests:
>
> test/lfc/data/alldata.lzx
> amazon
> calendar
>
>
>
>
> Deleted: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar
>
> Added: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
>
>
> Property changes on: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
> ___________________________________________________________________
> Name: svn:mime-type
> + application/octet-stream
>
> Modified: openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as
> ===================================================================
> --- openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as 2007-11-05 19:52:33 UTC (rev 7131)
> +++ openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as 2007-11-05 20:00:28 UTC (rev 7132)
> @@ -94,8 +94,10 @@
> */
> LzLoadQueue.XMLOnDataHandler = function (src) {
> if (src == undefined) {
> - Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL %w, no data received.", this.url);
> - Debug.warn("Failure to load data in serverless apps may be caused by Flash player security policies. Check your data server crossdomain.xml file");
> + if (!this.proxied) {
> + Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL %w, no data received.", this.url);
> + Debug.warn("Failure to load data in serverless apps may be caused by Flash player security policies. Check your data server crossdomain.xml file");
> + }
> this.onload(false);
> //Debug.write("this.loader.onerror.ready =", this.loader.onerror.ready);
> if (this.loader.onerror.ready) {
>
> Modified: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
> ===================================================================
> --- openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java 2007-11-05 19:52:33 UTC (rev 7131)
> +++ openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java 2007-11-05 20:00:28 UTC (rev 7132)
> @@ -3,7 +3,7 @@
> * ****************************************************************************/
>
> /* J_LZ_COPYRIGHT_BEGIN *******************************************************
> -* Copyright 2001-2004 Laszlo Systems, Inc. All Rights Reserved. *
> +* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. *
> * Use is subject to license terms. *
> * J_LZ_COPYRIGHT_END *********************************************************/
>
> @@ -169,7 +169,7 @@
> org.openlaszlo.i18n.LaszloMessages.getMessage(
> DataSource.class.getName(),"051018-169", new Object[] {new Long(size)})
> );
> - res.setContentLength((int)size);
> + //res.setContentLength((int)size);
> }
>
> if (doClientCache) {
>
> Modified: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
> ===================================================================
> --- openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java 2007-11-05 19:52:33 UTC (rev 7131)
> +++ openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java 2007-11-05 20:00:28 UTC (rev 7132)
> @@ -614,215 +614,8 @@
> HTTPDataSource.class.getName(),"051018-592", new Object[] {new Integer(code), HttpStatus.getStatusText(code)});
> }
>
> - /**
> - * A class for holding on to results of an Http fetch.
> - *
> - * @author <a href="mailto:bloch at laszlosystems.com">Eric Bloch</a>
> - */
>
> - public static class HttpData extends Data {
>
> - /** response code */
> - public final int code;
> -
> - /** Http request */
> - public final HttpMethodBase request;
> -
> - private PatternMatcher pMatcher = new Perl5Matcher();
> - private static final Pattern charsetPattern;
> - private static final Pattern declEncodingPattern;
> - static {
> - try {
> - Perl5Compiler compiler = new Perl5Compiler();
> - charsetPattern = compiler.compile(";charset=([^ ]*)");
> - declEncodingPattern =
> - compiler.compile("[ \t\r\n]*<[?]xml .*encoding=[\"']([^ \"']*)[\"'] .*[?]>");
> - } catch (MalformedPatternException e) {
> - throw new RuntimeException(e.getMessage());
> - }
> - }
> -
> - /**
> - * @param r filled request
> - * @param c response code
> - */
> - public HttpData(HttpMethodBase r, int c) {
> - code = c;
> - request = r;
> - }
> -
> - /**
> - * @return true if the data was "not modified"
> - */
> - public boolean notModified() {
> - return code == HttpServletResponse.SC_NOT_MODIFIED;
> - }
> -
> - /**
> - * @return the lastModified time of the data
> - */
> - public long lastModified() {
> -
> - Header lastModifiedHdr = request.getResponseHeader(
> - LZHttpUtils.LAST_MODIFIED);
> -
> - if (lastModifiedHdr != null) {
> - String lm = lastModifiedHdr.getValue();
> - mLogger.debug(
> -/* (non-Javadoc)
> - * @i18n.test
> - * @org-mes="data with last modified at " + p[0]
> - */
> - org.openlaszlo.i18n.LaszloMessages.getMessage(
> - HTTPDataSource.class.getName(),"051018-655", new Object[] {lm})
> -);
> - long l = LZHttpUtils.getDate(lm);
> - // Truncate to nearest second
> - return ((l)/1000L) * 1000L;
> - } else {
> - mLogger.debug(
> -/* (non-Javadoc)
> - * @i18n.test
> - * @org-mes="data has no mod time"
> - */
> - org.openlaszlo.i18n.LaszloMessages.getMessage(
> - HTTPDataSource.class.getName(),"051018-667")
> -);
> - return -1;
> - }
> - }
> -
> - /**
> - * append response headers
> - */
> - public void appendResponseHeadersAsXML(StringBuffer xmlResponse) {
> -
> - Header[] hedz = request.getResponseHeaders();
> - for (int i = 0; i < hedz.length; i++) {
> - String name = hedz[i].getName();
> - if (LZHttpUtils.allowForward(name, null)) {
> - xmlResponse.append("<header name=\""+ XMLUtils.escapeXml( name ) + "\" "
> - + "value=\"" + XMLUtils.escapeXml( hedz[i].getValue() ) + "\" />");
> - }
> - }
> - }
> -
> - /**
> - * release any resources associated with this data
> - */
> - public void release() {
> - request.releaseConnection();
> - }
> -
> - /**
> - * @return mime type
> - */
> - public String getMimeType() {
> - Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_TYPE);
> - String contentType = "";
> - if (hdr != null) {
> - contentType = hdr.getValue();
> - }
> - mLogger.debug(
> -/* (non-Javadoc)
> - * @i18n.test
> - * @org-mes="content type: " + p[0]
> - */
> - org.openlaszlo.i18n.LaszloMessages.getMessage(
> - HTTPDataSource.class.getName(),"051018-710", new Object[] {contentType})
> -);
> - return contentType;
> - }
> -
> - /**
> - * @return string
> - */
> - public String getAsString() throws IOException {
> - byte rawbytes[] = request.getResponseBody();
> - if (rawbytes == null || rawbytes.length == 0) {
> - throw new InterruptedIOException("null http response body");
> - }
> - String encoding = "UTF-8";
> - String content = getMimeType();
> - // search for ;charset=XXXX in Content-Type header
> - if (pMatcher.matches(content, charsetPattern)) {
> - encoding = pMatcher.getMatch().group(1);
> - }
> - // search for 'encoding' attribute in xml declaration, e.g.,
> - // <?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
> -
> - String decl = getXMLDeclaration(rawbytes);
> - if (pMatcher.matches(decl, declEncodingPattern)) {
> - encoding = pMatcher.getMatch().group(1);
> - //mLogger.debug("parsed data encoding: " + encoding);
> - }
> -
> - return new String(rawbytes, encoding);
> - }
> -
> - /** Returns the first non-whitespace line.
> - *
> - */
> - String getXMLDeclaration(byte buf[]) {
> - String str = new String(buf);
> - BufferedReader br = new BufferedReader(new StringReader(str));
> - String line;
> - while (true) {
> - try { line = br.readLine(); } catch (IOException e) { return ""; }
> - if (line == null) {
> - return "";
> - }
> - if (line.length() == 0) continue;
> - if (line.startsWith("<?xml ")) {
> - return line;
> - } else {
> - return "";
> - }
> - }
> - }
> -
> - /**
> - * @return input stream
> - */
> - public InputStream getInputStream() throws IOException {
> - InputStream str = request.getResponseBodyAsStream();
> - if (str == null) {
> - throw new IOException(
> -/* (non-Javadoc)
> - * @i18n.test
> - * @org-mes="http response body is null"
> - */
> - org.openlaszlo.i18n.LaszloMessages.getMessage(
> - HTTPDataSource.class.getName(),"051018-774")
> -);
> - }
> - return str;
> - }
> -
> - /**
> - * @return size, if known
> - */
> - public long size() {
> - Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_LENGTH);
> - if (hdr != null) {
> - String contentLength = hdr.getValue();
> - if (contentLength != null) {
> - mLogger.debug(
> -/* (non-Javadoc)
> - * @i18n.test
> - * @org-mes="content length: " + p[0]
> - */
> - org.openlaszlo.i18n.LaszloMessages.getMessage(
> - HTTPDataSource.class.getName(),"051018-794", new Object[] {contentLength})
> -);
> - int cl = Integer.parseInt(contentLength);
> - return cl;
> - }
> - }
> - return -1;
> - }
> - }
> -
> public static int getConnectionPoolTimeout() {
> return mConnectionPoolTimeout;
> }
>
> Added: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
>
>
> Property changes on: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
> ___________________________________________________________________
> Name: svn:executable
> + *
> Name: svn:mime-type
> + text/plain
> Name: svn:eol-style
> + native
>
> Modified: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
> ===================================================================
> --- openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java 2007-11-05 19:52:33 UTC (rev 7131)
> +++ openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java 2007-11-05 20:00:28 UTC (rev 7132)
> @@ -8,7 +8,7 @@
> * ****************************************************************************/
>
> /* J_LZ_COPYRIGHT_BEGIN *******************************************************
> -* Copyright 2001-2006 Laszlo Systems, Inc. All Rights Reserved. *
> +* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. *
> * Use is subject to license terms. *
> * J_LZ_COPYRIGHT_END *********************************************************/
>
> @@ -23,6 +23,7 @@
> import org.apache.log4j.*;
>
> import org.openlaszlo.utils.ContentEncoding;
> +import org.openlaszlo.utils.LZHttpUtils;
>
> import org.openlaszlo.media.MimeType;
> import org.openlaszlo.server.LPS;
> @@ -35,7 +36,14 @@
> import org.jdom.input.SAXBuilder;
> import org.jdom.output.XMLOutputter;
>
> +import org.apache.commons.httpclient.*;
> +import org.apache.commons.httpclient.methods.*;
> +import org.apache.commons.httpclient.util.*;
>
> +import org.xmlpull.v1.*;
> +
> +
> +
> /**
> * XML Converter
> *
> @@ -43,10 +51,33 @@
> public class XMLGrabber extends Converter {
>
> private static Logger mLogger = Logger.getLogger(XMLGrabber.class);
> + private static XmlPullParserFactory factory = null;
>
> + private static XmlPullParserFactory getXPPFactory () {
> + if (factory == null) {
> + // Set up the XML Parser factory
> + try {
> + String sys = null;
> + try {
> + sys = System.getProperty(XmlPullParserFactory.PROPERTY_NAME);
> + } catch (SecurityException se) {
> + }
> + factory = XmlPullParserFactory.newInstance(sys, null);
> + factory.setNamespaceAware(false);
> + factory.setValidating(false);
> + } catch (XmlPullParserException e) {
> + throw new RuntimeException(e.getMessage());
> + }
> + }
> + return factory;
> + }
> +
> /**
> * Convert incoming XML to ... XML
> *
> + * This method is called convertToSWF for historical reasons, and nobody
> + * has changed the API call name yet.
> + *
> * A dataset will look like this:
> * <resultset>
> * <body>
> @@ -61,63 +92,193 @@
> * </headers>
> * </resultset>
> */
> +
> public InputStream convertToSWF(Data data, HttpServletRequest req,
> HttpServletResponse res)
> throws ConversionException, IOException {
>
> - String body = data.getAsString();
> + try {
> + PipedOutputStream pout = new PipedOutputStream();
> + PipedInputStream in = new PipedInputStream(pout);
>
> - // Get headers
> - String sendheaders = req.getParameter("sendheaders");
> - StringBuffer headerbuf = new StringBuffer();
> - headerbuf.append("<headers>\n");
> - if (sendheaders == null || sendheaders.equals("true") ) {
> - data.appendResponseHeadersAsXML(headerbuf);
> - }
> - headerbuf.append("</headers>");
> - String headers = headerbuf.toString();
> + XmlSerializer serializer;
> + XmlPullParser parser;
> + parser = getXPPFactory().newPullParser();
> + InputStream dstream = data.getInputStream();
> + parser.setInput( dstream, null );
> + serializer = factory.newSerializer();
> + serializer.setOutput(pout , "UTF-8");
>
> - if (mLogger.isDebugEnabled()) {
> - mLogger.info("Output:" + body.length());
> - mLogger.info("Output:\n" + body);
> - mLogger.info("Output Headers:" + headers.length());
> - mLogger.info("Output Headers:\n" + headers);
> + HttpMethodBase request = ((HttpData) data).getRequest();
> +
> + final String sendheaders = req.getParameter("sendheaders");
> +
> + XMLCopyThread worker = new XMLCopyThread(pout, parser,serializer, request, sendheaders);
> + worker.start();
> +
> + return in;
> +
> + } catch (XmlPullParserException ex) {
> + throw new ConversionException("Parsing XML: " + ex.getMessage());
> }
> + }
>
> - // Default to true, for back compatibility (sigh)
> - boolean trimWhitespace = true;
> - String trimval = req.getParameter("trimwhitespace");
> - if ("false".equals(trimval)) {
> - trimWhitespace = false;
> + // Worker thread to parse XML (which serves to translate obscure
> + // charsets to UTF-8) and wrap it in <resultset>, possibly adding
> + // proxied HTTP headers from backedn response.
> + // This is written to the PipedOutputStream which we were passed.
> + class XMLCopyThread extends Thread implements Runnable {
> + OutputStream pout = null;
> + XmlPullParser parser;
> + XmlSerializer serializer;
> + HttpMethodBase request;
> + String sendheaders = null;
> +
> + XMLCopyThread( OutputStream pout, XmlPullParser parser, XmlSerializer serializer,
> + HttpMethodBase request,
> + String sendheaders) {
> + this.pout = pout;
> + this.parser = parser;
> + this.serializer = serializer;
> + this.request = request;
> + this.sendheaders = sendheaders;
> }
>
> - boolean compress = "true".equals(req.getParameter("compress"));
> -
> - // nsprefix now defaults to true
> - boolean nsprefix = true;
> - if ("false".equals(req.getParameter("nsprefix"))) {
> - nsprefix = false;
> + public void run() {
> + try {
> + writeXMLDataToOutputStream();
> + pout.flush();
> + pout.close();
> + } catch (XmlPullParserException ex) {
> + throw new RuntimeException(ex);
> + } catch (IOException ex) {
> + throw new RuntimeException(ex);
> + }
> }
>
> - // Need to parse body into a DOM, and add the headers, then re-emit as XML
> - // of the form
> + // Generates an XML document with this structure
> // <resultset>
> - // <body> ... </body>
> + // <body> [PROXIED_XMLDOC] </body>
> // <headers> ... </headers>
> // </resultset>
> + void writeXMLDataToOutputStream() throws XmlPullParserException, IOException {
> + //Run through XML PULL parser, to convert to UTF8, and
> + // wrap in <resultset> tag, plus optional headers
>
> - StringBuffer xdata = new StringBuffer();
> - xdata.append("<resultset>\n");
> - xdata.append("<body>\n");
> - String xbody = body.replaceFirst("<[?]xml.*?[?]>","");
> - xdata.append(xbody);
> - xdata.append("</body>\n");
> - xdata.append(headers);
> - xdata.append("</resultset>\n");
> + // Start a standalone document;
> + //serializer.startDocument("UTF-8", Boolean.TRUE);
>
> - // generate inputstream from DOM
> - ByteArrayInputStream dis = new ByteArrayInputStream((xdata.toString()).getBytes("UTF-8"));
> - return dis;
> + serializer.startTag("", "resultset");
> + serializer.startTag("", "body");
> +
> + parser.nextToken(); // read first token
> +
> + while (parser.getEventType () != XmlPullParser.END_DOCUMENT) {
> + writeToken ( parser.getEventType () );
> + parser.nextToken ();
> + }
> +
> + serializer.endTag("", "body");
> +
> + // <headers> ... </headers>
> + serializer.startTag("", "headers");
> +
> + // Get headers
> + if (sendheaders == null || sendheaders.equals("true") ) {
> + Header[] hedz = request.getResponseHeaders();
> + for (int i = 0; i < hedz.length; i++) {
> + String name = hedz[i].getName();
> + if (LZHttpUtils.allowForward(name, null)) {
> + serializer.startTag("", "header");
> +
> + serializer.attribute (null, "name", name);
> + serializer.attribute (null, "value", hedz[i].getValue());
> + serializer.endTag("", "header");
> + }
> + }
> + }
> + serializer.endTag("", "headers");
> +
> + serializer.endTag("", "resultset");
> + serializer.endDocument();
> + }
> +
> + private void writeStartTag ()
> + throws XmlPullParserException, IOException {
> + if (!parser.getFeature (XmlPullParser.FEATURE_REPORT_NAMESPACE_ATTRIBUTES)) {
> + for (int i = parser.getNamespaceCount (parser.getDepth ()-1);
> + i <= parser.getNamespaceCount (parser.getDepth ())-1; i++) {
> + serializer.setPrefix
> + (parser.getNamespacePrefix (i),
> + parser.getNamespaceUri (i));
> + }
> + }
> + serializer.startTag(parser.getNamespace (), parser.getName ());
> +
> + for (int i = 0; i < parser.getAttributeCount (); i++) {
> + serializer.attribute
> + (parser.getAttributeNamespace (i),
> + parser.getAttributeName (i),
> + parser.getAttributeValue (i));
> + }
> + }
> +
> +
> + private void writeToken (int eventType)
> + throws XmlPullParserException, IOException {
> + switch (eventType) {
> +
> + case XmlPullParser.START_TAG:
> + writeStartTag ();
> + break;
> +
> + case XmlPullParser.END_TAG:
> + serializer.endTag(parser.getNamespace (), parser.getName ());
> + break;
> +
> + case XmlPullParser.START_DOCUMENT:
> + //use Boolean.TRUE to make it standalone
> + //Boolean standalone = (Boolean) parser.getProperty(PROPERTY_XMLDECL_STANDALONE);
> + //serializer.startDocument(parser.getInputEncoding(), standalone);
> + break;
> +
> + case XmlPullParser.END_DOCUMENT:
> + //serializer.endDocument();
> + break;
> +
> + case XmlPullParser.IGNORABLE_WHITESPACE:
> + //comment it to remove ignorable whtespaces from XML infoset
> + String s = parser.getText ();
> + serializer.ignorableWhitespace (s);
> + break;
> +
> + case XmlPullParser.TEXT:
> + serializer.text (parser.getText ());
> + break;
> +
> + case XmlPullParser.ENTITY_REF:
> + serializer.entityRef (parser.getName ());
> + break;
> +
> + case XmlPullParser.CDSECT:
> + serializer.cdsect( parser.getText () );
> + break;
> +
> + case XmlPullParser.PROCESSING_INSTRUCTION:
> + // serializer.processingInstruction( parser.getText ());
> + break;
> +
> + case XmlPullParser.COMMENT:
> + //serializer.comment (parser.getText ());
> + break;
> +
> + case XmlPullParser.DOCDECL:
> + // serializer.docdecl (parser.getText ());
> + break;
> + }
> + }
> +
> +
> }
>
> /**
>
>
> _______________________________________________
> Laszlo-checkins mailing list
> Laszlo-checkins at openlaszlo.org
> http://www.openlaszlo.org/mailman/listinfo/laszlo-checkins
--
Regards,
Max Carlson
OpenLaszlo.org
More information about the Laszlo-dev
mailing list