Markus Rechberger / Perl
  
 StartSeite | MarkusRechberger/ | Neues | TestSeite | ForumSeite | Teilnehmer | Kategorien | Index | Hilfe | Einstellungen | Ändern
Perl Demo

This script fetches a list of airports and names, it also tries to figure out the corresponding countrycode. Finally it stores the data in a postgresql database for further tests.

(just includes a few perl topics as demonstration, no need to code it that 'strange') 

#!/bin/perl -w
use strict;

use LWP::Simple;
use Locale::Country;
use DBI;
use DBD::Pg;

my $dbh = DBI->connect("DBI:Pg:dbname=sampledb","postgres","");

sub demo::insertrow{
        shift;
        my ($cityairport,$countrycode,$code,$dbh) = @_;
        my $sth;
        my $query="select count(*) from countrycodes where code='$code'";
        $sth=$dbh->prepare($query);
        $sth->execute;
        if(($sth->fetchrow)[0]==0){
                $query="insert into countrycodes(cityairport,countrycode,code) values (?,?,?)";
                $sth=$dbh->prepare($query);
                $sth->execute($cityairport,$countrycode,$code);
        } else {
                print "already inserted ($code), skipping\n";
        }
}

my $dba;
my ($content,$line);
my $countrycode;
my $query;
my ($cityairport,$country,$code,$dummy);
my $pos;
my @content_array;
my @countrycodes;
$dba = "demo";
my @alphabet= ('a'..'w','xyz');
my $letter;
for(@alphabet){
        $letter=$_;
        $content = get("http://www.nationsonline.org/oneworld/airport_code_$letter.htm");
        print "processing http://www.nationsonline.org/oneworld/airport_code_$letter.htm\n";

        $pos=index($content,"<b>City, Airport</b>");
        $content=substr($content,$pos);
        $pos=index($content,"</tr>")+5;
        $content=substr($content,$pos);
        $pos=index($content,"</table>");
        $content=substr($content,0,$pos);
        $content =~ s/<[a-z\/]{2,3}>//g;

        foreach $line ( split(/<tr bgcolor="#EEEEEE">/,$content)){
                chomp($line);
                $line=~s/<.*>//g;
                if(length($line)>5){
                        ($dummy,$cityairport,$country,$code) = split(/\n/,$line);
                        $countrycode=country2code($country);
                        if(!$countrycode){
                                $countrycode="??";
                        }
                        if(!$code){
                                print "$line\n";
                        }
                        $code=~s/ //g;
                        $dba->insertrow($cityairport,$countrycode,$code,$dbh);
                }
        }
}

postgresql schema

--
-- PostgreSQL database dump
--

SET client_encoding = 'LATIN9';
SET check_function_bodies = false;

SET SESSION AUTHORIZATION 'postgres';

--
-- TOC entry 4 (OID 2200)
-- Name: public; Type: ACL; Schema: -; Owner: postgres
--

REVOKE ALL ON SCHEMA public FROM PUBLIC;
GRANT ALL ON SCHEMA public TO PUBLIC;


SET SESSION AUTHORIZATION 'postgres';

SET search_path = public, pg_catalog;

--
-- TOC entry 5 (OID 32682)
-- Name: countrycodes; Type: TABLE; Schema: public; Owner: postgres
--

CREATE TABLE countrycodes (
    cityairport character varying(255),
    countrycode character varying(10),
    code character varying(10) NOT NULL
);


--
-- TOC entry 6 (OID 42468)
-- Name: countrycode2; Type: INDEX; Schema: public; Owner: postgres
--

CREATE INDEX countrycode2 ON countrycodes USING btree (countrycode);


--
-- TOC entry 7 (OID 32684)
-- Name: countrycodes_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
--

ALTER TABLE ONLY countrycodes
    ADD CONSTRAINT countrycodes_pkey PRIMARY KEY (code);


--
-- TOC entry 3 (OID 2200)
-- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: postgres
--

COMMENT ON SCHEMA public IS 'Standard public schema';

preparing the parsed data for a C application

This script will generate a c header file with the airportlist in an optimized way 

#!/bin/perl -w
use strict;

use DBI;
use DBD::Pg;

my ($cityairport,$countrycode,$code);
my $codeindex;

my $dbh = DBI->connect("DBI:Pg:dbname=sampledb","postgres","");

my $sth=$dbh->prepare("select distinct countrycode from countrycodes order by countrycode asc");
$sth->execute;

my $sth2;
$codeindex="const char indexlist[]=\"";
my $airportindex="const int airportindex[]={0,";
my $airportlist="const char airportlist[][3]={";
my $i=0;
while(($countrycode)=$sth->fetchrow){
        $codeindex.="$countrycode,";
        $sth2=$dbh->prepare("select * from countrycodes where countrycode='$countrycode'");
        $sth2->execute;
        while(($cityairport,$countrycode,$code)=$sth2->fetchrow){
                $i++;
                $airportlist.="\"$code\",";
        }
        $airportindex.="$i,";


}
$codeindex=~s/.$//;
$airportindex=~s/.$//;
$airportlist=~s/.$//;
$codeindex.="\";";
$airportindex.="};";
$airportlist.="};";

open(HANDLE,">dbtable.h");
print HANDLE $codeindex."\n";
print HANDLE $airportindex."\n";
print HANDLE $airportlist."\n";
close(HANDLE);

Speed comparison

Comparing the duration of 5000 retrieves/queries

1.) C

(not optimized c code) 

#include <stdio.h>
#include "dbtable.h"
#include "corelib.h"

int corelib_search(char *request,char ***input){
        char *pos;
        char **inarray=0;
        int position;
        int i;
        int arraysize=0;
        int elements=1;
        pos=strstr(indexlist,request);
        if(pos>0){
//              printf("%d\n",pos-indexlist);
                position=(pos-indexlist)/3;
//              printf("startpos: %d %d\n",airportindex[position],airportindex[position+1]);
                inarray=(char**)malloc(elements*sizeof(char**));
                inarray[0]=0;
                for(i=airportindex[position];i<airportindex[position+1];i++){
                        inarray=(char**)realloc(inarray,(++elements)*sizeof(char**));
                        inarray[elements-2]=(char*)malloc(strlen(airportlist[i])+1);
                        strcpy(inarray[elements-2],airportlist[i]);
                        inarray[elements-1]=0;
//                      printf("-> %s\n",airportlist[i]);


                }
        }
        (*input)=inarray;
        return(0);
}

int corelib_free(char ***pointer){
        int i=0;
        while((*pointer)[i]!=0){
                free((*pointer)[i]);
                i++;
        }
        free(*pointer);
}

int corelib_print_r(char ***pointer){
        int i=0;
        while((*pointer)[i]!=0){
                printf("-> %s\n",(*pointer)[i]);
                i++;
        }
        return(0);
}

char **corelib(char* argument){
        char **output=0;
        corelib_search(argument,&output);
        return(output);
}

corelib.h 
__BEGIN_DECLS //__BEGIN_DECLS is nothing else than a wrapper for C++ projects which might use this library (also see extern "C")
int corelib_search(char *request,char ***);
int corelib_free(char ***pointer);
int corelib_print_r(char ***pointer);
char **corelib(char* argument);
__END_DECLS

plain Makefile 
CC=gcc
LIBTOOL=libtool
LIBS=-lcorelib
LIBDIR=-L`pwd`
OUTFILE=-o testapp
all:    corelib testapp
corelib:
        $(LIBTOOL) --mode=link $(CC) -shared corelib.c -o libcorelib.so -g
testapp:
        $(CC) testapp.c $(LIBS) $(LIBDIR) $(OUTFILE) -g

testapp.c 

#include <stdio.h>
#include "corelib.h"

int main(){
        int i;
        char **retval=0;
        for(i=0;i<5000;i++){
                retval=corelib("us");
//              corelib_print_r(&retval);
                corelib_free(&retval);
        }
        return(0);
}

real    0m0.739s
user    0m0.671s
sys     0m0.001s

2.) Perl (native)

following script will generate a script called perlscript.pl 

#!/bin/perl -w
use strict;

use DBI;
use DBD::Pg;

my ($cityairport,$countrycode,$code);
my $codeindex;
my $dbh = DBI->connect("DBI:Pg:dbname=sampledb","postgres","");
my $sth=$dbh->prepare("select countrycode,code from countrycodes order by countrycode asc");
$sth->execute;
$codeindex="const char indexlist[]=\"";
my $airportindex="const int airportindex[]={0,";
my $airportlist="const char airportlist[][3]={";
my $i=0;
my %hashdb;
my $index;
my $index2;
open(HANDLE,">perlscript.pl");
print HANDLE <<EOF;
#!/bin/perl -w
use strict;
my \$index2;
my \$index;
my \$i;
EOF
print HANDLE "my %hashdb = (\n";

while(($countrycode,$code)=$sth->fetchrow){
        $codeindex.="$countrycode,";
        $hashdb{"$countrycode"}{$code}=0;
        print HANDLE "     \"$countrycode\" => { \"$code\" => 0},\n";
}
print HANDLE "             );\n";
print HANDLE <<EOF;
for(\$i=0;\$i<=5000;\$i++){
        foreach \$index (keys \%hashdb){
                if(\$index eq "us"){
                        foreach \$index2 (keys \%{ \$hashdb{\$index}}){
#                               print "\$index2"."\\n";
                        }
                }
                last if \$index eq "us";
        }
}
EOF
$sth->finish;
close HANDLE;

time perl perlscript.pl

real    0m0.678s
user    0m0.590s
sys     0m0.004s

3.) Perl/Mysql? 

#!/bin/perl -w
use strict;

use LWP::Simple;
use Locale::Country;
use DBI;
use DBD::Pg;

my ($cityairport,$countrycode,$code);
my $codeindex;

my $dbh = DBI->connect("DBI:mysql:sampledb","root","");

my $sth=$dbh->prepare("select code from countrycodes where countrycode='us'");
for(my $b=0;$b<=5000;$b++){
        $sth->execute;
}
$sth->finish;
$dbh->disconnect;

time perl testspeed.pl 

real    0m4.754s
user    0m1.471s
sys     0m2.423s

4.) Perl/Postgresql? 

#!/bin/perl -w
use strict;

use LWP::Simple;
use Locale::Country;
use DBI;

my ($cityairport,$countrycode,$code);
my $codeindex;

my $dbh = DBI->connect("DBI:Pg:dbname=sampledb","postgres","");

my $sth=$dbh->prepare("select code from countrycodes where countrycode='us'");
for(my $b=0;$b<=5000;$b++){
        $sth->execute;
}
$sth->finish;
$dbh->disconnect;

time perl testspeed.pl

real    0m16.001s
user    0m1.295s
sys     0m0.084s

Perl C Module (using perlxs)

1. ccsample.xs 

#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#include "../corelib/corelib.h"

MODULE = ccsample               PACKAGE = ccsample

void returncountry(arg)
        SV  *arg
    PPCODE:
        int argn=0;
        int i=0;
        char **retval=0;
        retval=corelib("us");
        if(retval!=0){
                while(retval[i]!=NULL){
                        XPUSHs(sv_2mortal(newSVpvf(retval[i])));
                        argn++;
                        i++;
                }
        }
        corelib_free(&retval);

2. ccsample.pl 
package ccsample;

use strict;
use warnings;

require Exporter;
require DynaLoader;

our @ISA = qw(Exporter DynaLoader);
our @EXPORT = qw(
);
our $VERSION = '0.01';

bootstrap ccsample $VERSION;

1;
__END__

3. Makefile.PL 
use ExtUtils::MakeMaker;
WriteMakefile(
    NAME         => 'ccsample',
    VERSION_FROM => 'ccsample.pm',
    LIBS         => ['-lcorelib'],
);

4. build it 

# perl Makefile.PL
Writing Makefile for ccsample
# make
...
# make install
Files found in blib/arch: installing files in blib/lib into architecture dependent library tree
Writing /usr/local/lib/perl/5.8.4/auto/ccsample/.packlist
Appending installation info to /usr/local/lib/perl/5.8.4/perllocal.pod

5. perl testfile 

#!/bin/perl -w
use ccsample;
for($i=0;$i<5000;$i++){
        @footest=ccsample::returncountry("us");
}

real    0m4.178s
user    0m4.033s
sys     0m0.003s

StartSeite | MarkusRechberger/ | Neues | TestSeite | ForumSeite | Teilnehmer | Kategorien | Index | Hilfe | Einstellungen | Ändern
Text dieser Seite ändern (zuletzt geändert: 22. Juni 2005 22:35 (diff))
Suchbegriff: gesucht wird
im Titel
im Text