Headlines News :

POPULAR POST

Showing posts with label linux. Show all posts
Showing posts with label linux. Show all posts

Squid Youtube Cache 2016 Terbaru suport HTTP & HTTPS (Squid 3.5.3)



apt-get update && apt-get upgrade -y

apt-get install devscripts \
build-essential \
openssl libssl-dev \
fakeroot \
libcppunit-dev \
libsasl2-dev \
cdbs \
ccze \
libfile-readbackwards-perl \
libcap2 \
libcap-dev \
libcap2-dev \
libtool \
sysv-rc-conf -y &&
wget http://ftp.riken.jp/net/squid/archive/3.5/squid-3.5.3.tar.bz2 &&
tar -xjf squid-3.5.3.tar.bz2 &&
cd squid-3.5.3 &&
./configure \
--prefix=/usr \
--includedir=/usr/include \
--infodir=/usr/share/info \
--sysconfdir=/etc \
--localstatedir=/var \
--libexecdir=/usr/lib/squid \
--srcdir=. \
--datadir=/usr/share/squid \
--sysconfdir=/etc/squid \
--mandir=/usr/share/man \
--enable-inline \
--enable-async-io=24 \
--enable-storeio=ufs,aufs,diskd,rock \
--enable-removal-policies=lru,heap \
--enable-gnuregex \
--enable-delay-pools \
--enable-cache-digests \
--enable-underscores \
--enable-icap-client \
--enable-follow-x-forwarded-for \
--enable-eui \
--enable-esi \
--enable-icmp \
--enable-zph-qos \
--enable-http-violations \
--enable-ssl-crtd \
--enable-linux-netfilter \
--enable-ltdl-install \
--enable-ltdl-convenience \
--enable-x-accelerator-vary \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-translation \
--disable-ipv6 \
--disable-ident-lookups \
--with-swapdir=/var/spool/squid \
--with-logdir=/var/log/squid \
--with-pidfile=/var/run/squid.pid \
--with-aufs-threads=24 \
--with-filedescriptors=65536 \
--with-large-files \
--with-maxfd=65536 \
--with-openssl \
--with-default-user=proxy \
--with-included-ltdl &&
make && make install


mkdir /var/lib/squid &&
chown -R nobody /var/lib/squid/ &&
/usr/lib/squid/ssl_crtd -c -s /var/lib/squid/ssl_db &&
chown -R proxy:proxy /var/lib/squid/ssl_db/ &&
chmod -R 777 /var/lib/squid/ssl_db/

nano /etc/squid/squid.conf

#
# Recommended minimum configuration:
#

# Example rule allowing access from your local networks.
# Adapt to list your (internal) IP networks from where browsing
# should be allowed
acl localnet src 192.168.10.0/24 #LAN
acl localnet src 10.10.10.0/24 #WIFI
acl localnet src 10.10.20.0/24 #WIFI
acl localnet src 10.10.30.0/24 #WIFI

acl SSL_ports port 443
acl Safe_ports port 80  # http
acl Safe_ports port 21  # ftp
acl Safe_ports port 443  # https
acl Safe_ports port 70  # gopher
acl Safe_ports port 210  # wais
acl Safe_ports port 1025-65535 # unregistered ports
acl Safe_ports port 280  # http-mgmt
acl Safe_ports port 488  # gss-http
acl Safe_ports port 591  # filemaker
acl Safe_ports port 777  # multiling http
acl CONNECT method CONNECT

# TAG: QUERY
# -----------------------------------------------------------------------------
acl QUERY urlpath_regex -i (hackshield|blank.html|infinity.js|hshield.da|renew_session_token.php|recaptcha.js|dat.asp|notice.swf|patchlist.txt|hackshield|captcha|reset.css|update.ver|notice.html|updates.txt|gamenotice|images.kom|patchinfo.xml|noupdate.ui|\.Xtp|\.htc|\.txt)
acl QUERY urlpath_regex -i (patch.conf|uiimageset.xml.iop|gashaponwnd.xml.iop|loading.swf|download.swf|version.list|version.ini|launch.jnlp|server_patch.cfg.iop|core.swf|Loading.swf|resouececheck.sq|mainloading.swf|config.xml|gemmaze.swf|xml.png|size.xml|resourcesbar.swf|version.xml|version.list|delete.ini)
acl QUERY urlpath_regex -i \.(jsp|asp|aspx|cfg|iop|zip|php|xml|html)(\?|$)
cache deny QUERY

#
acl dontstore url_regex ^http:\/\/(([\d\w-]*(\.[^\.\-]*?\..*?))(\/\mosalsal\/[\d]{4}\/.*\/)(.*\.flv))\?start.*
acl dontstore url_regex redbot\.org \.php
acl dontstore url_regex -i ^http:\/\/.*gemscool\.com\/.*
acl dontstore url_regex \.(aspx|php)\?
acl dontstore url_regex goldprice\.org\/NewCharts\/gold\/images\/.*\.png
acl dontstore url_regex google\.co(m|\.[a-z]{2})\/complete\/search\?
acl dontstore url_regex redirector\.([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com)\/(get_video\?|videodownload\?|videoplayback.*id|get_video_info\?|ptracking\?|player_204\?|stream_204\?).*

acl store_yt_id url_regex -i youtube.*(ptracking|stream_204|playback|player_204|watchtime|set_awesome|s\?|ads).*(video_id|docid|\&v|content_v)\=([^\&\s]*).*$
acl store_id_list_yt url_regex -i (youtube|googlevideo).*videoplayback.*$
acl store_id_list_yt url_regex ^https?\:\/\/([0-9.]{4}|.*\.youtube\.com|.*\.googlevideo\.com|.*\.video\.google\.com)\/(get_video\?|videodownload\?|videoplayback.*id).*

acl store-id_list urlpath_regex -i dl\.sourceforge\.net
acl store-id_list urlpath_regex -i \.ytimg\.com
acl store-id_list urlpath_regex -i \.(akamaihd|fbcdn)\.net
acl store_id_list urlpath_regex -i [a-zA-Z]{2}[0-9]*\.4shared\.com\/download\/

acl store_id_list_url url_regex ^http:\/\/[0-9]\.bp\.blogspot\.com.*\.(jpeg|jpg|png|gif|ico)
acl store_id_list_url url_regex ^http[s]?:\/\/.*\.twimg\.com\/(.*)\.(gif|jpeg|jpg|png|js|css)
acl store_id_list_url url_regex ^http[s]?:\/\/(media|static)\.licdn\.com\/.*\.(png|jpg|gif|woff)
acl store_id_list_url url_regex ^https:\/\/fb(static|cdn)\-.*\-a.akamaihd.net\/(.*)\.(gif|jpeg|jpg|png|js|css|mp4)
acl store_id_list_url url_regex ^http:\/\/.*\.ak\.fbcdn\.net\/.*\.(gif|jpg|png|js|mp4)

request_header_access Range deny store_id_list_yt
range_offset_limit 10 KB store_id_list_yt

acl loop_302 http_status 302
acl getmethod method GET

###############################################################################
# Recommended minimum Access Permission configuration:
#
# Deny requests to certain unsafe ports
###############################################################################
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow localhost manager
http_access deny manager
http_access allow localnet
http_access allow localhost
http_access deny all

###############################################################################
# squid ssl_bump option
###############################################################################
always_direct allow all
ssl_bump server-first all
sslproxy_cert_error deny all
sslproxy_flags DONT_VERIFY_PEER

sslcrtd_program /usr/lib/squid/ssl_crtd -s /var/lib/squid/ssl_db -M 4MB
sslcrtd_children 8 startup=1 idle=1

###############################################################################
# Squid normally listens to port 3128
###############################################################################
https_port 3130 tproxy ssl-bump generate-host-certificates=on dynamic_cert_mem_cache_size=4MB cert=/etc/squid/ssl_certs/squid.crt key=/etc/squid/ssl_certs/squid.key
http_port 3129 tproxy
http_port 3128

# TAG: Store-id Program
# -----------------------------------------------------------------------------
store_id_program /etc/squid/store-id.pl
store_id_children 100 startup=0 idle=1 concurrency=1000

# TAG: Store-id Access
# -----------------------------------------------------------------------------
store_id_access deny dontstore
store_id_access deny !getmethod
store_id_access allow store_id_list_yt
store_id_access allow store_yt_id
store_id_access allow store-id_list
store_id_access deny all
store_id_bypass on

# TAG: Youtube 302
# -----------------------------------------------------------------------------
store_miss deny store_id_list_yt loop_302
send_hit deny store_id_list_yt loop_302

###############################################################################
## MEMORY CACHE OPTIONS
###############################################################################
client_dst_passthru on
cache_mem 1024 MB
maximum_object_size_in_memory 1024 KB
memory_cache_shared off
memory_cache_mode disk
memory_replacement_policy heap GDSF

###############################################################################
## DISK CACHE OPTIONS
###############################################################################
cache_replacement_policy heap LFUDA
minimum_object_size 1 bytes
maximum_object_size 10 GB

###############################################################################
# Uncomment and adjust the following to add a disk cache directory.
###############################################################################
cache_dir aufs /cache-1 500000 16 256 # sesuaikan dengan drive penyimpanan cache
cache_dir aufs /cache-2 500000 16 256 # sesuaikan dengan drive penyimpanan cache
store_dir_select_algorithm round-robin
cache_swap_low 90
cache_swap_high 95

###############################################################################
# Leave coredumps in the first cache dir
###############################################################################
coredump_dir /var/spool/squid

###############################################################################
## LOGFILE OPTIONS
###############################################################################
#access_log daemon:/tmp/access.log !log
access_log /tmp/access.log squid
logfile_daemon /usr/lib/squid/log_file_daemon
cache_store_log none
logfile_rotate 1
mime_table /etc/squid/mime.conf
pid_filename /var/run/squid.pid
strip_query_terms off
buffered_logs off

###############################################################################
## OPTIONS FOR TROUBLESHOOTING
###############################################################################
#cache_log /tmp/cache.log
cache_log /dev/null
#debug_options ALL,1 22,3
coredump_dir /var/spool/squid

###############################################################################
## OPTIONS FOR TUNING THE CACHE
###############################################################################
max_stale 1 years
vary_ignore_expire on
shutdown_lifetime 10 seconds

###############################################################################
# Add any of your own refresh_pattern entries above these.
###############################################################################
refresh_pattern ^ftp:  1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0

# Youtube Video
refresh_pattern -i (get_video\?|videoplayback\?|videodownload\?|\.mp4|\.webm|\.flv|((audio|video)\/(webm|mp4))) 241920 100% 241920 override-expire ignore-reload ignore-private ignore-no-store ignore-must-revalidate reload-into-ims ignore-auth store-stale
refresh_pattern -i ^https?\:\/\/.*\.googlevideo\.com\/videoplayback.*     10080 99% 43200 override-lastmod override-expire ignore-reload reload-into-ims ignore-private reload-into-ims ignore-auth store-stale
refresh_pattern -i ^https?\:\/\/.*\.googlevideo\.com\/videoplayback.*$    241920 100% 241920 override-expire ignore-reload ignore-private ignore-no-store ignore-must-revalidate reload-into-ims ignore-auth store-stale

# Image Youtube
refresh_pattern -i (yimg|twimg)\.com\.*         1440 100% 129600 override-expire ignore-reload reload-into-ims
refresh_pattern -i (ytimg|ggpht)\.com\.*        1440 80% 129600 override-expire override-lastmod ignore-auth ignore-reload reload-into-ims

#images facebook
refresh_pattern -i fbcdn.*net\/.*\.((jp(e?g|e|2)|gif|pn[pg]|bm?|tiff?|ico|swf|css|js)|(jp(e?g|e|2)|gif|pn[pg]|bm?|tiff?|ico|swf|css|js)(\?|.*$)) 241920 99% 241920 ignore-no-store ignore-private override-expire override-lastmod reload-into-ims ignore-auth
refresh_pattern -i pixel\.facebook\.com.*\.(jpg|png|gif|ico|css|js) 241920 80% 241920 override-expire ignore-reload reload-into-ims ignore-auth
refresh_pattern -i \.akamaihd\.net.*\.(jpg|png|gif|ico|css|js) 241920 80% 241920 override-expire ignore-reload reload-into-ims ignore-auth
refresh_pattern -i ((facebook.com)|(85.131.151.39))\.(jpg|png|gif) 241920 99% 241920 ignore-reload override-expire ignore-no-store store-stale
refresh_pattern -i fbcdn\.net\/.*\.((jp(e?g|e|2)|gif|pn[pg]|bm?|tiff?|ico|swf|css|js)|(jp(e?g|e|2)|gif|pn[pg]|bm?|tiff?|ico|swf|css|js)(\?|.*$)) 241920 99% 241920 ignore-no-store ignore-private override-expire override-lastmod reload-into-ims ignore-auth
refresh_pattern static\.(xx|ak)\.fbcdn\.net*\.(jpg|gif|png) 241920 99% 241920 ignore-reload override-expire ignore-no-store
refresh_pattern ^https?\:\/\/profile\.ak\.fbcdn.net*\.(jpg|gif|png) 241920 99% 241920 ignore-reload override-expire ignore-no-store

# Video Facebook
refresh_pattern -i \.video.ak.fbcdn.net.*\.(mp4|flv|mp3|amf)                    10080 80% 43200 override-expire ignore-reload reload-into-ims ignore-private ignore-no-store ignore-must-revalidate
refresh_pattern (audio|video)\/(webm|mp4) 129600 99% 129600 ignore-reload override-expire override-lastmod ignore-must-revalidate  ignore-private ignore-no-store ignore-auth store-stale
refresh_pattern -i ^http://.*squid\.internal.*  241920 100% 241920 override-lastmod override-expire ignore-reload ignore-must-revalidate ignore-private ignore-no-store ignore-auth store-stale

# All File
refresh_pattern -i \.(3gp|7z|ace|asx|bin|deb|divx|dvr-ms|ram|rpm|exe|inc|cab|qt) 10080 80% 10080 override-expire override-lastmod reload-into-ims
refresh_pattern -i \.(rar|jar|gz|tgz|bz2|iso|m1v|m2(v|p)|mo(d|v)|arj|lha|lzh|zip|tar|iop|nzp|pak|mar|msp) 10080 80% 10080 override-expire override-lastmod reload-into-ims ignore-reload
refresh_pattern -i \.(jp(e?g|e|2)|gif|pn[pg]|bm?|tiff?|ico|swf|dat|ad|txt|dll) 10080 80% 10080 override-expire override-lastmod reload-into-ims
refresh_pattern -i \.(avi|ac4|mp(e?g|a|e|1|2|3|4)|mk(a|v)|ms(i|u|p)|og(x|v|a|g)|rm|r(a|p)m|snd|vob|webm) 10080 80% 10080 override-expire override-lastmod reload-into-ims
refresh_pattern -i \.(pp(t?x)|s|t)|pdf|rtf|wax|wm(a|v)|wmx|wpl|cb(r|z|t)|xl(s?x)|do(c?x)|flv|x-flv) 10080 80% 10080 override-expire override-lastmod reload-into-ims
refresh_pattern .  0 20% 4320

###############################################################################
## ADMINISTRATIVE PARAMETERS
###############################################################################
cache_mgr eko.hendratno@gmail.com
cache_effective_user proxy
cache_effective_group proxy
visible_hostname gtw.home.lan
unique_hostname gtw.home.lan

###############################################################################
## PERSISTENT CONNECTION HANDLING
###############################################################################
detect_broken_pconn on
client_persistent_connections off
server_persistent_connections on

###############################################################################
## ERROR PAGE OPTIONS
###############################################################################
error_directory /usr/share/squid/errors/en
error_log_languages off

###############################################################################
## DNS OPTIONS
###############################################################################
check_hostnames off
hosts_file /etc/hosts
connect_retries 2
ipcache_low 90
ipcache_high 95
ipcache_size 10024                        # 2x Besar RAM
fqdncache_size 7024                        # real RAM Hardware
pipeline_prefetch 100

###############################################################################
## MISCELLANEOUS
###############################################################################
memory_pools off
reload_into_ims on
uri_whitespace strip
max_filedescriptors 65536



=================================
nano /etc/squid /store-id.pl
=================================
#!/usr/bin/perl
###########################
#
# Store-ID dengan asumsi chanel berapapun
#
###########################
$|=1;
while (<>) {
my $chan = "";
if (s/^(\d+\s+)//o) {
$chan = $1;
}
$_ =~ s/(\s+.+)//o;

if ($_ =~ m/^https?\:\/\/.*youtube.*(ptracking|stream_204|player_204|gen_204).*(video_id|docid|v)\=([^\&\s]*).*/){
        $vid = $3 ;
        @cpn = m/[&?]cpn\=([^\&\s]*)/;
                $fn = "/var/log/squid3/@cpn";
                unless (-e $fn) {
                        open FH,">".$fn ;
                        print FH "$vid\n";
                        close FH;
                }
        print $chan, "ERR\n" ;

} elsif ($_ =~ m/^https?\:\/\/.*(youtube|google).*videoplayback.*/){
        @itag = m/[&?](itag=[0-9]*)/;
        @ids = m/[&?]id\=([^\&\s]*)/;
        @mime = m/[&?](mime\=[^\&\s]*)/;
        @cpn = m/[&?]cpn\=([^\&\s]*)/;
        @range = m/[&?](range=[^\&\s]*)/;
        if (defined($cpn[0])) {
            $fn = "/var/log/squid3/@cpn";
            if (-e $fn) {
                open FH,"<".$fn ;
                $id  = ;
                chomp $id ;
                close FH ;
                  } else {
                $id = $ids[0] ;
            }
        print $chan, "OK store-id=http://googlevideo.squid.internal/id=" . $id . "&@itag@range@mime\n" ;
        } else {
        print $chan, "ERR\n" ;
        }

} elsif ($_ =~ m/^http:\/\/(videos|photos|scontent)[\-a-z0-9\.]*instagram\.com\/hphotos[\-a-z0-9]*\/([\w\d\-\_\/\.]*.(mp4|jpg))/){
        print $chan, "OK store-id=http://instagram.squid.internal/$2\n" ;
} elsif ($_ =~ m/^http:\/\/distilleryimage[\-a-z0-9\.]*instagram\.com\/(.*)/){
        print $chan, "OK store-id=http://instagram.squid.internal/$1\n" ;

} elsif ($_ =~ m/^https?:\/\/.*\.steampowered\.com\/depot\/[0-9]+\/chunk\/([^\?]*)/){
        print $chan, "OK store-id=http://steampowered.squid.internal/$1\n" ;

} elsif ($_ =~ m/^https?:\/\/.*(fbcdn|akamaihd)\.net\/.*\/(.*\.mp4)(.*)/) {
        print $chan, "OK store-id=storeurl://facebook.squid.internal/$2\n" ;

} elsif ($_ =~ m/^https?:\/\/.*(static|profile).*a\.akamaihd\.net(\/static-ak\/rsrc\.php\/v[0-9]\/(.*\.(mp4|jpg|bmp|png|flv|m4v|gif|jpeg)))/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/.*(static|profile).*\.ak\.fbcdn\.net(\/static-ak\/rsrc\.php\/v[0-9]\/(.*\.(mp4|jpg|bmp|png|flv|m4v|gif|jpeg)))/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/.*(static|profile).*a\.akamaihd\.net(\/rsrc\.php\/v[0-9]\/(.*))/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/.*(static|profile).*\.ak\.fbcdn\.net(\/rsrc\.php\/v[0-9]\/(.*))/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*(fbcdn|akamaihd)[^\/]*net\/rsrc\.php\/(.*\.(mp4|jpg|bmp|png|flv|m4v|gif|jpeg))/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$2\n" ;

} elsif ($_ =~ m/^https?:\/\/[^\/]*(fbcdn|akamaihd)[^\/]*net\/safe\_image\.php\?.*(url\=.*\.(mp4|jpg|bmp|png|flv|m4v|gif|jpeg)).*/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/i[0-2].wp\.com\/graph\.facebook\.com\/(.*)/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/(video\.ak\.fbcdn\.net)\/(.*?)\/(.*\.mp4)\??.*$/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$1/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/video\.(.*)\.fbcdn\.net\/(.*?)\/([0-9_]+\.(mp4|flv|avi|mkv|m4v|mov|wmv|3gp|mpg|mpeg)?)(.*)/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/(fbcdn|scontent).*(akamaihd|fbcdn)\.net\/(h|s)(profile|photos).*\/((p|s).*\.(png|gif|jpg))(\?.+)?$/){
        print $chan, "OK store-id=http://facebook.squid.internal/$5\n" ;
} elsif ($_ =~ m/^https?:\/\/(fbcdn|scontent).*(akamaihd|fbcdn)\.net\/(h|s)(profile|photos).*\/(.*\.(png|gif|jpg))(\?.+)?$/){
        print $chan, "OK store-id=http://facebook.squid.internal/$5\n" ;

} elsif ($_ =~ m/^https?:\/\/attachment\.fbsbx\.com\/.*\?(id=[0-9]*).*/) {
        print $chan, "OK store-id=http://facebook.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https:\/\/.*\.google\.com\/chrome\/win\/.+\/(.*\.exe)/){
        print $chan, "OK store-id=http://update-google.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.ytimg\.com\/(.*\.(webp|jpg|gif))/){
        print $chan, "OK store-id=http://ytimg.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*firedrive\.com\/download\/[0-9]+\/[0-9]+\/.*\?h=.*e\=.*f\=(.*)\&.*/){
        print $chan, "OK store-id=http://firedrive.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.4shared\.com\/.*\/dlink__[23]F([\w]+)_[23]F(.*)\_3Ftsid_[\w].*/){
        print $chan, "OK store-id=http://4shared.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.4shared\.com\/download\/([^\/]*).*/){
        print $chan, "OK store-id=http://4shared.squid.internal/$1\n" ;

} elsif ($_ =~ m/^https?:\/\/.*\.[a-z]+\.bing\.net\/(.*)\&w=.*/){
        print $chan, "OK store-id=http://bing.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.bing\.(net|com)\/.*\?id=([a-zA-Z]\.[0-9]+)&pid=.*/){
        print $chan, "OK store-id=http://bing.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.gstatic\.com\/images\?q=tbn\:(.*)/){
        print $chan, "OK store-id=http://gstatic.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.reverbnation\.com\/.*\/(ec_stream_song|download_song_direct|stream_song)\/([0-9]*).*/){
        print $chan, "OK store-id=http://reverbnation.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.dl\.sourceforge\.net\/(.*\.(exe|zip|mp3|mp4))/){
        print $chan, "OK store-id=http://sourceforge.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/fs[0-9]+\.filehippo\.com\/[^\/]*\/[^\/]*\/(.*)/){
        print $chan, "OK store-id=http://filehippo.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/download[0-9]+.mediafire\.com\/.*\/\w+\/(.*)/){
        print $chan, "OK store-id=http://mediafire.squid.internal$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*android\.clients\.google\.com\/[a-z]+\/[a-zA-Z]+\/[a-zA-Z]+\/(.*)\/([0-9]+)\?.*/){
        print $chan, "OK store-id=http://android.squid.internal/$1/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/.*(googleusercontent.com|blogspot.com)\/(.*)\/([a-z0-9]+)(-[a-z]-[a-z]-[a-z]+)?\/(.*\.(jpg|png))/){
        print $chan, "OK store-id=http://googleusercontent.squid.internal/$5\n" ;
} elsif ($_ =~ m/^https?:\/\/global-shared-files-[a-z][0-9]\.softonic\.com\/.{3}\/.{3}\/.*\/.*\=(.*\.exe)/){
        print $chan, "OK store-id=http://softonic.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*netmarble\.co\.id\/.*\/(data|ModooMarble)\/(.*)/){
        print $chan, "OK store-id=http://netmarble.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/(.*)\.windowsupdate\.com\/(.*)\/(.*)\/([a-z].*)/){
        print $chan, "OK store-id=http://windowsupdate.squid.internal/$4\n" ;
} elsif ($_ =~ m/^https?:\/\/.*filetrip\.net\/.*\/((.*)\.([^\/\?\&]{2,4}))\?.*$/){
        print $chan, "OK store-id=http://filetrip.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*get4mobile\.net\/.*f=([^\/\?\&]*).*$/){
        print $chan, "OK store-id=http://get4mobile.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*thestaticvube\.com\/.*\/(.*)/){
        print $chan, "OK store-id=http://thestaticvube.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/113\.6\.235\.171\/youku\/.*\/(.*\.flv)/){
        print $chan, "OK store-id=http://youku.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/\d+\.\d+\.\d+\.\d+\/drama\/(.*\.mp4)\?.*\=(\d+)/){
        print $chan, "OK store-id=http://drama.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/([a-z])[\d]{1,2}?(.gstatic\.com.*|\.wikimapia\.org.*)/){
        print $chan, "OK store-id=http://gstatic.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.[a-z][0-9]\.(tiles\.virtualearth\.net)\/(.*\&n=z)/){
        print $chan, "OK store-id=http://virtualearth.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/imgv2-[0-9]\.scribdassets\.com\/(.*)/){
        print $chan, "OK store-id=http://scribdassets.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/(.*?)\/(archlinux\/[a-zA-Z].*\/os\/.*)/){
        print $chan, "OK store-id=http://archlinux.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/(.*?)\/speedtest\/(.*\.(jpg|txt))\??.*$/){
        print $chan, "OK store-id=http://speedtest.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/i[1-9]{3}\.photobucket\.com\/(.*)/){
        print $chan, "OK store-id=http://photobucket.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/i[1-9]{4}\.photobucket\.com\/(.*)/){
        print $chan, "OK store-id=http://photobucket.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/avideos\.5min\.com\/.*\/(.*)\?.*/){
        print $chan, "OK store-id=http://avideos.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.catalog\.video\.msn\.com\/.*\/(.*\.(mp4|flv|m4v))/){
        print $chan, "OK store-id=http://msn-video.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/v\.imwx\.com\/.*\/(.*)\?.*/){
        print $chan, "OK store-id=http://imwx.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/video[0-9]\.break\.com\/.*\/(.*)\?.*/){
        print $chan, "OK store-id=http://break.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.video[0-9]\.blip\.tv\/.*\/(.*)\?.*/){
        print $chan, "OK store-id=http://blip.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/ss[0-9]\.vidivodo\.com\/vidivodo\/vidservers\/server[0-9]*\/videos\/.*\/([a-zA-Z0-9.]*)\?.*/){
        print $chan, "OK store-id=http://vidivodo.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/video\-http\.media\-imdb\.com\/([a-zA-Z0-9\@\_\-]+\.(mp4|flv|m4v))\?.*/){
        print $chan, "OK store-id=http://imdb-video.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/(vl|v)\.mccont\.com\/(.*)\/(.*\.(mp4|m4v|flv))\?.*/){
        print $chan, "OK store-id=http://mccont.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/(vid.{0,2}|proxy.*)(\.ak|\.ec|\.akm|)\.(dmcdn\.net|dailymotion\.com)\/.*\/(frag.*\.(flv|mp4|m4v)).*/){
        print $chan, "OK store-id=http://dailymotion.squid.internal/$4\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.vimeo[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg|web))\?.*/){
        print $chan, "OK store-id=http://vimeo.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/fcache\.veoh\.com\/.*\/.*(l[0-9]*\.(mp4|flv))\?.*/){
        print $chan, "OK store-id=http://veoh.squid.internal$1\n" ;
} elsif ($_ =~ m/^https?:\/\/video\.thestaticvube\.com\/.*\/(.*)/){
        print $chan, "OK store-id=http://thestaticvube.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/cdn[0-9]\.videos\.videobash\.com\/.*\/(.*\.(mp4|m4v|flv))\?.*/){
        print $chan, "OK store-id=http://videobash.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.phncdn[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://phncdn.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.xvideos\.com\/.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://xvideos.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.tube8[^\/]*\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://tube8.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.(redtube|redtubefiles)\.com\/.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://redtube.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/.*\/xh.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))/){
        print $chan, "OK store-id=http://xhcdn.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.xhcdn[^\/]*\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://xhcdn.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.nsimg[^\/]*\.net.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://nsimg.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.youjizz\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://youjizz.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.public\.keezmovies[^\/]*\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://keezmovies.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.youporn[^\/]*\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://youporn.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.spankwire[^\/]*\.com.*\/([^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://spankwire.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.pornhub[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://pornhub.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.us.playvid[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://playvid.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.slutload-media[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://slutload-media.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.hardsextube[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://hardsextube.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*\.public\.extremetube[^\/]*\.com.*\/([[^\/]*\.(flv|mp4|avi|mkv|mp3|rm|rmvb|m4v|mov|wmv|3gp|mpg|mpeg))\?.*/){
        print $chan, "OK store-id=http://extremetube.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/([a-z0-9.]*)(\.doubleclick\.net|\.quantserve\.com|.exoclick\.com|interclick.\com|\.googlesyndication\.com|\.auditude\.com|.visiblemeasures\.com|yieldmanager|cpxinteractive)(.*)/){
        print $chan, "OK store-id=http://ads.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/(.*?)\/(ads)\?(.*?)/){
        print $chan, "OK store-id=http://ads.squid.internal/$3\n" ;
} elsif ($_ =~ m/^https?:\/\/[^\/]*phobos\.apple\.com\/.*\/([^\/]*\.ipa)/){
        print $chan, "OK store-id=http://apple.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/fs\w*\.fileserve\.com\/file\/(\w*)\/[\w-]*\.\/(.*)/){
        print $chan, "OK store-id=http://fileserve.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/s[0-9]*\.filesonic\.com\/download\/([0-9]*)\/(.*)/){
        print $chan, "OK store-id=http://filesonic.squid.internal/$2\n" ;
} elsif ($_ =~ m/^https?:\/\/download[0-9]{3}\.avast\.com\/(.*)/){
        print $chan, "OK store-id=http://avast.squid.internal/41\n" ;
} elsif ($_ =~ m/^https?:\/\/[a-zA-Z0-9]+\.[a-zA-Z0-9]+x\.[a-z]\.avast\.com\/[a-zA-Z0-9]+x\/(.*\.vpx)/){
        print $chan, "OK store-id=http://avast.squid.internal\$1\n" ;
} elsif ($_ =~ m/^https?:\/\/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/(iavs.*)/){
        print $chan, "OK store-id=http://iavs.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/.*\.starhub\.com\/[a-z]+\/[a-z]+\/[a-z]+\/(.*exe)\?[0-9]/){
        print $chan, "OK store-id=http://starhub.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/dnl-[0-9]{2}\.geo\.kaspersky\.com\/(.*)/){
        print $chan, "OK store-id=http://kaspersky.squid.internal/$1\n" ;
} elsif ($_ =~ m/^https?:\/\/([^\.]*)\.yimg\.com\/(.*)/){
        print $chan, "OK store-id=http://yimg.squid.internal/$1\n" ;
} else {
        print $chan, "ERR\n" ;
}
}
=================================
nano /etc/init.d/squid
=================================
#! /bin/sh
#
# squid        Startup script for the SQUID HTTP proxy-cache.
#
# Version:    @(#)squid.rc  2.20  01-Oct-2001  miquels@cistron.nl
#
### BEGIN INIT INFO
# Provides:          squid
# Required-Start:    $local_fs $network
# Required-Stop:     $local_fs $network
# Should-Start:      $named
# Should-Stop:       $named
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Squid HTTP Proxy
### END INIT INFO

NAME=squid
DAEMON=/usr/sbin/squid
LIB=/usr/lib/squid
PIDFILE=/var/run/squid.pid
# export LD_PRELOAD=/usr/local/lib/libjemalloc.so # ini apabila anda menggunakan jemalloc
SQUID_ARGS="-YC"

[ ! -f /etc/default/squid ] || . /etc/default/squid

. /lib/lsb/init-functions

PATH=/bin:/usr/bin:/sbin:/usr/sbin

[ -x $DAEMON ] || exit 0

grepconf () {
    w="     " # space tab
    sq=/etc/squid/squid.conf
    # sed is cool.
    res=`sed -ne '
        s/^'$1'['"$w"']\+\([^'"$w"']\+\).*$/\1/p;
        t end;
        d;
        :end q' < $sq`
    [ -n "$res" ] || res=$2
    echo "$res"
}

grepconf2 () {
    w="     " # space tab
    sq=/etc/squid/$NAME.conf
    # sed is cool.
    res=`sed -ne '
        s/^'$1'['"$w"']\+[^'"$w"']\+['"$w"']\+\([^'"$w"']\+\).*$/\1/p;
        t end;
        d;
        :end q' < $sq`
    [ -n "$res" ] || res=$2
    echo "$res"
}

#
#    Try to increase the # of filedescriptors we can open.
#
maxfds () {
    [ -n "$SQUID_MAXFD" ] || return
    [ -f /proc/sys/fs/file-max ] || return 0
    global_file_max=`cat /proc/sys/fs/file-max`
    minimal_file_max=$(($SQUID_MAXFD + 4096))
    if [ "$global_file_max" -lt $minimal_file_max ]
    then
        echo $minimal_file_max > /proc/sys/fs/file-max
    fi
    ulimit -n $SQUID_MAXFD
}

start () {
    cdr=`grepconf2 cache_dir /cache-1`
    ctp=`grepconf cache_dir ufs`

    case "$cdr" in
        [0-9]*)
            log_failure_msg "squid: squid.conf contains 2.2.5 syntax - not starting!"
            log_end_msg 1
            exit 1
            ;;
    esac
   
    #
    # Create spool dirs if they don't exist.
    #
    if [ -d "$cdr" -a ! -d "$cdr/00" ] || [ "$ctp" = "coss" -a ! -w "$cdr" ]
    then
        log_warning_msg "Creating squid cache structure"
        $DAEMON $SQUID_ARGS -z
    fi

    if [ "$CHUID" = "" ]; then
        CHUID=root
    fi

    maxfds
    umask 027
    start-stop-daemon --quiet --start \
        --pidfile $PIDFILE \
        --chuid $CHUID \
        --exec $DAEMON -- $SQUID_ARGS < /dev/null
    return $?
}

stop () {
    PID=`cat $PIDFILE 2>/dev/null`
    start-stop-daemon --stop --quiet --pidfile $PIDFILE --name squid
    #
    #    Now we have to wait until squid has _really_ stopped.
    #
    sleep 2
    if test -n "$PID" && kill -0 $PID 2>/dev/null
    then
        log_action_begin_msg " Waiting"
        cnt=0
        while kill -0 $PID 2>/dev/null
        do
            cnt=`expr $cnt + 1`
            if [ $cnt -gt 24 ]
            then
                log_action_end_msg 1
                return 1
            fi
            sleep 5
            log_action_cont_msg ""
        done
        log_action_end_msg 0
        return 0
    else
        return 0
    fi
}

case "$1" in
    start)
    log_daemon_msg "Starting Squid HTTP proxy" "squid"
    if start ; then
        log_end_msg $?
    else
        log_end_msg $?
    fi
    ;;
    stop)
    log_daemon_msg "Stopping Squid HTTP proxy" "squid"
    if stop ; then
        log_end_msg $?
    else
        log_end_msg $?
    fi
    ;;
    reload|force-reload)
    log_action_msg "Reloading Squid configuration files"
    $DAEMON -k reconfigure
    log_action_end_msg 0
    ;;
    restart)
    log_daemon_msg "Restarting Squid HTTP proxy" "squid"
stop
    if start ; then
        log_end_msg $?
    else
        log_end_msg $?
    fi
    ;;
    status)
    status_of_proc -p "$PIDFILE" "$DAEMON" squid && exit 0 || exit $?
    ;;
    *)
    echo "Usage: /etc/init.d/$NAME {start|stop|reload|force-reload|restart|status}"
    exit 3
    ;;
esac

exit 0


=================================
chmod +x store-id.pl
chmod +x squid
chown proxy:proxy /cache-1
chown proxy:proxy /cache-2 &&
chmod 777 /cache-1 &&
chmod 777 /cache-2
squid -f /etc/squid/squid.conf -z

sysv-rc-conf squid default

=================================
nano /etc/rc.local
=================================
#0
iptables -A POSTROUTING -t nat -j MASQUERADE -o eth0
#1
iptables -t mangle -F
iptables -t mangle -X
#2
echo 0 > /proc/sys/net/ipv4/conf/lo/rp_filter
echo 1 > /proc/sys/net/ipv4/ip_forward
#3
ip rule add fwmark 1 lookup 100
ip route add local 0.0.0.0/0 dev lo table 100
#4
iptables -t mangle -N DIVERT
iptables -t mangle -A PREROUTING -p tcp -m socket -j DIVERT
#5
iptables -t mangle -A DIVERT -j MARK --set-mark 1
iptables -t mangle -A DIVERT -j ACCEPT
#6
iptables -t mangle -A PREROUTING -p tcp --dport 80 -j TPROXY --tproxy-mark 0x1/0x1 --on-port 3129
iptables -t mangle -A PREROUTING -p tcp --dport 443 -j TPROXY --tproxy-mark 0x1/0x1 --on-port 3130
#7
squid start
exit 0
===================================================================================================
squid -k parse
squid -k reconfigure
squid -z
squid start

tail -f /tmp/access.log | ccze

Hadoop dan Cara Instalasi-nya


Hadoop adalah salah satu implementasi dari konsep Map Reduce yang bersifat Open Sorce untuk memproses model atau data berskala besar secara terdistribusi. Hadoop diimplementasikan di JAVA dengan mengunakan class library, dengan menggunakan Hadoop data akan diolah secara simultan, melalui lebih dari satu mesin (terdistribusi). Map Reduce sendiri ialah sebuah teknik  untuk menangani masalah pengaturan pada data berskala besar, hasil yang diharapkan agar data dapat diolah menjadi data yang reliable dan relevan.

Hadoop dapat digunakan secara Open Source, yaitu dapat digunakan dan dikembangkan untuk kepentingan bersama. Hadoop dapat diunduh langsung dari situs resminya di :

Sebelum menjalankan Hadoop pastikan anda telah menginstall Java Jdk seperti diuraikan pada bab 3.1. Kemudian  tempatkan folder hadoop pada /opt/ dan lakukan beberapa konfigurasi seperti di bawah ini.
a)      Install openssh pada setiap komputer.
Secure Shell (ssh) adalah protocol untuk komunikasi client-server untuk mengakses shell secara remote.Semua pertukaran pesan (message exchange) akan di enkripsi menggunakan enkripsi 128bit. Algorithm yang paling sering digunakan adalah rsa, namun juga bisa menggunakan algorithm enkripsiasynchronouslainnya.
SSH client dan server yang paling populer dan akan digunakan dalam dokumentasi ini adalah OpenSSH, berikut ini adalah cara instalasi OpenSSH pada linux server distro Debian dan CentOS.
Debian:
su #
apt-get install openssh-server openssh-client
Kemudian ikuti langkah instalasi.
CentOS / Redhat / Fedora:
su #
yum install openssh-server
yum install openssh-client
b)      Konfigurasi key ssh untuk setiap computer
Konfigurasi ini berguna agar komunikasi Secure Shell (ssh) antar computer tidak memerlukan authentikasipassword lagi.Agar dapat berkomunikasi tanpa authentikasi, public keyssh clientharus didaftarkan kedalam file authorized_keys server.
Gunakan perintah berikut pada terminal pada setiap komputer yang akan dijadikan hadoop node.
·         ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa 
·         cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
·         chmod -R 700 .ssh/
Untuk melakukan tes gunakan perintah =>ssh localhost pada terminal, bila konfigurasi di atas berhasil maka tidak ada lagi permintaan untuk memasukan password.
Cara ini hanya dapat digunakan untuk Virtual Machine (VM)yang dibuat dengan cara di clone. Hal ini disebabkan karena hardware yang dibuat dengan cara di clone memiliki hardware-hardware ID yang sama, yang perlu digunakan untuk authentikasi public key SSH.
c)       Konsep Hadoop
Hadoop memiliki beberapa terminologi yang akan digunakan didalam dokumentasi ini:
HDFS Hadoop File System adalah DistributedFile System untuk hadoop, bedanya dengan File System biasa adalah replikasi. HDFS memiliki jumlah replikasi, setiap file didalam HDFS akan di replikasi sesuai dengan jumlah replikasi pada setting.
Map Reduceadalah konsep pembagian job atau proses pada sistem distribusi hadoop.Proses pendistribusian proses terdapat dalam Map dan pada Reduce terdapat proses pengabungan output atau hasil.
Baik Map Reduce dan HDFS pada hadoop menggunakan konsep master slave.Master adalahkomputer yang mendistribusikan proses dan juga berfungsi sebagai gateway antara sistem distribusi hadoop dan aplikasi. Slave adalah komputer yang mengerjakan proses yang didistribusikan oleh master node.
d)      Setting Konfigurasi Hadoop (Mapred & HDFS Master digabung)
Sebagai contoh, kita akan melakukan instalasi Hadoop pada 4 buah mesin.Pada konfigurasi ini Map Reduce cluster dan HDFS cluster akan digabugkan, sehingga hanya terdapat 1 master node. Dengan konfigurasi ini,  1 Virtual Machine (VM) akan difungsikan sebagai master, dan 3 VM menjadi Slave dengan pengaturan  IP  sebagai berikut:
·         komputer 1 (sebagai master)        = 192.168.1.1
·         komputer 2 (sebagai slave)         = 192.168.1.2
·         komputer 3 (sebagai slave)         = 192.168.1.3
·         komputer 4 (sebagai slave)         = 192.168.1.4
Setelah langkah diatas selesai, maka lakukan konfigurasi Hadoop yang berada pada folder /opt/hadoop/conf. Di dalam folder conf tersebut ada beberapa berkas yang perlu kita setting ulang, yaitu pada berkas
1.       masters,
2.  slaves,
3.       core-site.xml(Setting hadoop HDFS Mster dan behavior HDFS)
4.       mapred-site.xml (Setting hadoop mapred master dan behavoiour mapred process).
Penjelasan perubahan Setting :
·         Pada komputer 1 atau master hadoop bukalah berkas master dengan perintah => vim /opt/hadoop/conf/masters dan masukkan ip address dari hadoop master yang sudah anda set sebelumnya. Perhatikan pada contoh dibawah.
           192.168.1.1
·         Pada komputer 1 atau master hadoop bukalah berkas slaves dengan perintah => vim /opt/hadoop/conf/slaves dan masukkan ip addressdari hadoop slave yang sudah anda set sebelumnya. Perhatikan pada contoh dibawah.
           192.168.1.1
           192.168.1.2
           192.168.1.3
           192.168.1.4
·         Pada komputer 1 atau master hadoop bukalah berkas core-site.xml dengan perintah => vim /opt/hadoop/conf/ core-site.xml dan editlah sesuai dengan text dibawah ini.
Sebagai perbandingan perhatikan lampiran berkas konfigurasi berikut ini, sesuaikan VM yang menjadi Master, dan VM yang menjadi Slaves.Kesalahan setting dapat menyebabkan Hadoop tidak dapat digunakan.
hadoop.tmp.dir
/data/hadoop/tmp
fs.default.name
hdfs://IP HADOOP MASTER:54310
io.compression.codecs
org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec
io.compression.codec.lzo.class
com.hadoop.compression.lzo.LzoCodec
fs.inmemory.size.mb
200


Penjelasan:
·      hadoop.tmp.dirdirektori yang digunakan oleh hadoop node sebagai blok file system.
·      fs.default.nameIP address HDFS Master node.
·      io.compression.codecskompresi data yang digunakan untuk mengurangi besar packet.
·      fs.inmemory.size.mbbesar cache yang di alokasikan didalam memory
·         Pada komputer 1 atau master hadoop bukalah berkas mapred-site.xml dengan perintah => vim /opt/hadoop/conf/mapred-site.xml dan editlah sesuai dengan text dibawah ini.


mapred.job.tracker
IP address Hadoop Master:54311
mapred.tasktracker.map.tasks.maximum
4
mapred.tasktracker.reduce.tasks.maximum
4
mapred.reduce.child.java.opts
-Xmn600m -Xms768m -Xmx768m
mapred.map.child.java.opts
-Xmn600m -Xms768m -Xmx768m
mapreduce.map.output.compress
true
mapreduce.map.output.compress.codec
com.hadoop.compression.lzo.LzoCodec
io.sort.mb
200
io.sort.spill.percent
0.95


Penjelasan:
·      mapred.job.trackerIP adress dan port hadoop mapred master node.
·      mapred.tasktracker.map.tasks.maximumjumlah maksimum map task yang boleh dijalankan node
·      mapred.tasktracker.reduce.tasks.maximumkompresi data yang digunakan untuk mengurangi besar packet.
·      mapred.reduce.child.java.optskonfigurasi java runtime, khususnya memori untuk reduce task.
·      mapred.map.child.java.optskonfigurasi java runtime, khususnya memori untuk maptask.
·      mapredreduce.map.output.compress compress output map atau tidak.
·      io.sort.mbbuffer memory saat sorting data (dari map ke reduce process).
·      io.spill.percent
e)      Copy semua config hadoop master node ke semua slave
Setiap hadoop hanya membutuhkan 1 set konfigurasi dengan dapat dengan mudah di copy atau disamakan antara master dan slave. Hal ini dapat dilakukan dengan 2 cara, cara pertama adalah menyalin (copy) folder /opt/hadoop/conf master ke seluruh slave atau menyalin (copy) seluruh folder hadoop ke setiap node slave.
Untuk mengcopy folder conf saja lakukan command ini untuk setiap IP SLAVE:
· scp-r /opt/hadoop/conf mypc@IP Adress_SLAVE1:/opt/hadoop/
·scp-r /opt/hadoop/conf mypc@IP_Adress SLAVE2:/opt/hadoop/
(lakukan untuk semua slave)
Untuk mengcopy seluruh folder hadoop, apabila folder hadoop belum ada di komputer slave, lakukan langkah ini pada setiap IP SLAVE:
scp -r /opt/hadoop root@IP_Adress SLAVE1:/opt/
(masukkan password root IP_SLAVE1)
(masukkan password root IP_SLAVE1)
chown -R ukp4:ukp4 /opt/hadoop/
(lakukkan untuk slave 1, 2, dan semuanya)
f)       Format berkas system hadoop
Berikut ini adalah perintah terminal untuk format berkas system Hadoop.
Lakukan proses ini untuk setiap hadoop node, baik master maupun slave.
·         Masuklah dengan super user pada komputer yang ingin diformat hadoop file systemnya dan buatlah forder denga perintah =>mkdir /data/hadoop. Folder ini akan berfungsi untuk menjadi tempat penyimpanan berkas system hadoop.
·         Ubahlah hak kepemilikan folder /data/hadoop dengan perintah =>chown mypc:mypc /data/hadoop
·         setelah langkah pertama dan kedua di jalankan barulah format hadoop bisa di lakukan dengan perintah =>/opt/hadoop/bin/hadoop namenode -format
·        
“Storage directory /data/hadoop/tmp/dfs/name has been successfully formatted.”Jika tulisan tersebut ada dalam terminal maka proses format telah berhasil 
g)      Menjalankan hadoop
Hadoop dapat dijalankan melalui semua node, namun lebih baik apabila untuk menjalankan Hadoop cukup dari komputer yang di set sebagai hadoop master saja. Berikut adalah perintah menjalankan hadoop untuk men-start HDFS cluster dan Mapred cluster:
·         /opt/hadoop/bin/start-all.sh
Selain itu dapat juga menstartHDFS dan mepred secara terpisah:
·         HDFS: /opt/hadoop/bin/start-dfs.sh
·         Mapred: /opt/hadoop/bin/start-mapred.sh
Setelah hadoop berjalan cobalah buka melalui browser apakah hadoop sudah berjalandengan benar atau belum dengan link sebagai berikut http://192.168.1.1:50030Pada tampilan dibrowser bisa dilihat informasi hadoop yang sedang berjalan, misalnya :
·         State adalah status Hadoopnya sekarang adalah running.
·         Nodes adalah jumlah komputer yang terdaftar pada sub-sistem administrasi Hadoop.
·         Running jobs adalah tempat dimana kita bisa melihat job apa yang sedang di kerjakan oleh Hadoop.
h)      Menghentikan Hadoop
Sama seperti saat menjalankan, saat menghentikan Hadoop juga lebih baik apabila dijalankan dari komputer yang di set sebagai master. Berikut adalah perintah  menghentikan hadoop cluster yang sedang berjalan (HDFS & Mapred):
·         /opt/hadoop/bin/stop-all.sh
Untuk hanya Menghentikan HDFS atau Mapred:
·         HDFS: /opt/hadoop/bin/stop-dfs.sh

·         Mapred: /opt/hadoop/bin/stop-mapred.sh
 
Copyright © 2011. COMASTNET - All Rights Reserved