Showing posts with label Linux. Show all posts
Showing posts with label Linux. Show all posts

Tuesday, 11 February 2014

How I downloaded large files from proxy server which imposed file-size limit.

In this post, I will be describing about one of the methods I discovered to download large files from a server which actually imposed file-size limit. So, Intuitively what do you think I am gonna do? I will be downloading it by parts. But how?

At first, I started testing how is the proxy server actually able to detect such requests. I found that it merely checks the length field of the incoming packet and throws an error if its size exceeds the maximum specified value.

Then I went through the HTTP Request Protocol. At some stage I came to know that there is a special field in the header called Range. With this we can actually request the start and end points of the bytes in the file in zero-based manner.

For example, if you want to download a file of size say 50 bytes(that's too tiny now-a-days). You only want bytes from 34 to 43 say. Then the HTTP request looks as follows:

GET file_name.extension HTTP/1.1
....
....
Range: bytes=33-42
...
...
It starts from 33 because it is zero-based, meaning the first byte starts at zero and so on. Now, I am able to figure out how to download a file by parts. But another question remains. What will be the size of the entire file? How to figure this out?

This problem was solved when I looked at the response from the server. For instance, the response looks as follows when the above request is sent:

HTTP/1.1 206 Partial content
....
....
Content-Range: bytes 33-42/50
Content-length: 10
...
...
I think you should be able to figure it out from the above response. The total length of the file is send in the Content-Range field after the "/". So, first I request only 1 byte of data, which then gives me the length of the file and then proceed further to download it by parts.

We are ready with the idea and its a matter of coding the above idea. I have used the urllib2 from python, since I was too lazy to code it in C.

As an extension to this, I have used threads which increased the speed to large extent and it was as if I downloaded from my Local Area Network rather than the Internet.

As you may argue whats so special about this? There are many download accelerators that employ this method. But wait a minute. None of them employ the exact method I have described above. Indeed, well-known programs such as axel,wget and others failed to download the file when such constraint was introduced. So, I guess my idea is a bit better given these circumstances :).

Here is the code for my idea in Python:

import urllib2,sys,thread,time,tempfile,os
data=[]
def partial_download(url, st, en,idv):
 global data
# print 'Thread:',str(idv),' for ',str(en-st+1),'bytes'
 req = urllib2.Request(url)
 req.headers["Range"]='bytes='+str(st)+'-'+str(en)
 f = urllib2.urlopen(req)
 fd = tempfile.NamedTemporaryFile(delete=False)
 resp = ''
 while 1: 
  stt = f.read()
  if not stt:
   break
  resp += stt
 fd.write(resp)
 fd.close()
 data.append([idv,fd])
 print 'Thread:',str(idv),'finished getting ',str(en-st+1),'bytes to',fd.name

if len(sys.argv)<3:
 print 'Format:[url] [parallel_download_count]'
 sys.exit()
parallel_download_count = 1


parallel_download_count = int(sys.argv[2])

proxy = urllib2.ProxyHandler({'http': 'http://172.30.0.19:3128'})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
link = sys.argv[1]
file_name = link.split("/")[-1:][0]
print file_name
#print link
req = urllib2.Request(link)
#first we need to know the content-length..

req.headers['Range'] = 'bytes=0-0'
f = urllib2.urlopen(req)
meta =f.info()
content_length = int(meta["Content-Range"].split('/')[1])
print 'File-size:',content_length

chunk_size = content_length/parallel_download_count

curr_count = 0
idc = 0
while curr_count+chunk_size<=content_length:
 thread.start_new_thread(partial_download, (link,curr_count, curr_count+chunk_size-1,idc))
 idc+=1
 curr_count += chunk_size

if curr_count+chunk_size>content_length:
 thread.start_new_thread(partial_download,(link,curr_count,content_length-1,idc))
 idc+=1

while len(data)<idc:
 time.sleep(1)

print 'Merging into single file...'
data.sort()
#file_type = meta['Content-Type'].split('/')[1]
fd =open(file_name, 'w')

for chunk in data:
 tmp_fd = open(chunk[1].name,'r')
 tmps = tmp_fd.read()
 fd.write(tmps)
 print 'Wrote',len(tmps),'bytes!'
 tmp_fd.close()
 os.unlink(chunk[1].name)
fd.close()

#print 'Length:',meta.getheaders("Content-Length")[0]


Thursday, 15 November 2012

Miscellaneous Programs

In this post, I added some interesting programs written in C to implement some commands .


Our very first program is to implement a Command Interpreter. The idea to implement this is simple. Just take the input command from the user and call exec in a forked child and print the output. The code is as follows:

#include<stdio.h>
#include<signal.h>
#include<unistd.h>
#include<stdlib.h>
#include<string.h>
#include<sys/types.h>
#include<fcntl.h>
#include<sys/stat.h>
#define MAX_ARGS 5
void runcmd()
{
 int c=0, take=0, give=0;
 char *i[MAX_ARGS],g;
 char *input_file=(char*)malloc(12*sizeof(char));
 char *output_file=(char*)malloc(12*sizeof(char));
 
 for(int j=0; j<MAX_ARGS; j++) i[j]=(char*)malloc(9*sizeof(char));
 do
 {
        /*read all the input into a char double pointer*/    
  scanf("%s",i[c++]);
  if(!strcmp(i[c-1], ">") || !strcmp(i[c-1], "<"))
  {
   if(!strcmp(i[c-1], ">"))
   {
                /*if the symbol is '>' then read the output
                 * file-name to which the data must  be sent
                 */
    scanf("%s",output_file);
    give=1;
   }
   else
   {
                /*if the symbol is '<', then we are gonna 
                 *have an input file to take data from
                 */
    scanf("%s", input_file);
    take=1;
   }
    
  c--;
  }
  /*read the nxt char of every string if its '\n'
         * then terminate the loop..
         */
  g=getc(stdin);
 }while(g!='\n');

 i[c]=NULL;

 pid_t h;

 if((h=fork()) < 0) printf("fork error\n");
 else if(h==0)
 {
  if(give==1)
  {
   int fd1 = open(output_file, O_RDWR | O_CREAT,  S_IRUSR | S_IWUSR|S_IRGRP);
   dup2(fd1,1);
  }
  else if(take==1)
  {
   int fd = open(input_file, O_RDWR ,  S_IRUSR | S_IWUSR|S_IRGRP);
   if(fd==-1)
   {
    printf("unsupported file or file doesn't exists\n");
    return;
   }
   dup2(fd,STDIN_FILENO);
  }
  execvp(i[0], i);
  /*if the control transfers here, it means 
   *execvp has not executed successfully
   */
  printf("unknown command\n");
 }
 else
 {
  waitpid(h, NULL, NULL);
        /*wait for the child to terminate*/ 
 }
} 
int main(int argc, char* argv[])
{
 while(1)
 {
 printf("-->"); 
 /*keep on calling this function*/
 runcmd();
    }
 exit(0);
}

The above program works for most of the commands and I also provide the stream re-direction facility, to transfer the data into/from files.


The second program is to implement the ps command. For those who don't know what this does, it simply displays the list of the processes running from the current terminal. If you append an argument '-A', it gives the list of all the processes running on the system. Our task is to implement the later one.

The idea is to open the proc directory. This contains numbered directories and some other directories. Each numbered directory denotes a running process and the number on the directory represents its process-ID.

Inside every numbered directory lies a file called "stat". This contains the information that is needed to display the necessary contents. For example, the stat file of init process is as follows:

nitish@nitish:~$ cat /proc/1/stat
1 (init) S 0 1 1 0 -1 4202752 14453 339956 23 20 19 35 257 129 20 0 1
 0 3 25030656 613 18446744073709551615 1 1 0 0 0 0 0 4096 536962595
 18446744073709551615 0 0 17 0 0 0 41 0 0 0 0 0 0 0 0 0 0

The first of these is the init's process ID, second is the name of the process, third is the process status, fourth is the parent process ID(scheduler in this case).

These strings will be displayed for each numbered directory and then we are done. The C code is as follows:

#include<stdio.h>
#include<stdlib.h<
#include<dirent.h<
#include<string.h<
#include<sys/types.h<
#include<unistd.h<


int main(int argc, char *argv[])
{
 DIR *d;
 char *t;
 int N=0,pid;
 int cnt=0;
 int curr[10], curr_cnt=0;
 struct dirent *fl;
 d=opendir("/proc");
 int M=getppid();
 printf("Process ID of the terminal : %d\n", M);
 printf("Format: PID  /  PNAME  /  STATE  /  PPID \n");
 while((fl=readdir(d))!=NULL)
 {
  int num=atoi(fl->d_name);
  N=0;
  if(num!=0)
  {
   cnt++;
   t= (char*)malloc(15*sizeof(char));
   strcpy(t,"/proc/");
   strncat(t,fl->d_name,10);
   strncat(t,"/stat",6);  
   FILE *ptr=fopen(t,"r");
   if(ptr==NULL)
   printf("error\n");
   char tt[20];
   char g;
   /**process id**/
   while((g=getc(ptr))!=' ')
   N=N*10 + (g-48);
   pid=N;
   printf("%d  /  ", N);
   g=getc(ptr);
   /**process name**/
   while((g=getc(ptr))!=')' )
   {
    printf("%c", g);
   }
   printf("  /  ");
   g=getc(ptr);
   g=getc(ptr);
   if(g=='S')
    printf("Interruptible sleep  /  ");
   else if(g=='D')
    printf("Uninterruptible sleep  /  ");
   else if(g=='R')
    printf("Runnable  /  ");
   else if(g=='T')
    printf("Stopped  /  ");
   else if(g=='X')
    printf("Dead  /  ");
   else if(g=='Z')
    printf("Zombie  /  ");
   else if(g=='W')
    printf("Paging  /  ");
   g=getc(ptr);
   N=0;
   while((g=getc(ptr))!=' ')
    N = N*10 + (g-48);
   printf("%d" , N);
   if(N==M)
   {
    printf("**\n");
    curr[curr_cnt++]=pid;
   }
   fclose(ptr);
   printf("\n");
  }
 }
 printf("**\nTOTAL PROCESSES:%d\n**\n", cnt);
 printf("**********\n");
 printf("Processes inside the terminal:%d\n",curr_cnt );
 for(int i=0; i<curr_cnt; i++)
  printf("%d ", curr[i]);
 printf("\n");
exit(0);
}

If you want to display only the processes specific to the current terminal, you need to just store all those process IDs which have parent process ID as terminal. Sounds simple isn't it?

The normal ps command displays some other information like tty and command name and these can also be found in each numbered directory which I leave it as an exercise.

The extension of this would be to display all the PIDs, hierarchically starting with the init process and also to find all the file names opened by a given process(of course you must explore the proc directory a bit to this one).


Lastly, I would like to post the code for the thread library, which I have described in my previous posts. Here is the code. (Don't panic by seeing the length, most of them were for indentations and comments).

#include<stdio.h>
#include<stdlib.h>
#include<signal.h>
#include<sys/types.h>
#include<sys/time.h>
#include<string.h>
#include<errno.h>
#include<ucontext.h>
#include "thqueue.c"
#define MAX_THREADS 10
#define STACK_SIZE 64000
#define TIME_SLOT 500000   //time slot given to each thread in round robin scheduling (0.25sec here..)
typedef struct Task* Tptr;

int CURRENT_THREAD_COUNT=0;
int TERMINATED=0;
int Global_id;


 ucontext_t Main;  //context for saving main...

 ucontext_t Delete[MAX_THREADS];  //context for thread recycling...
 int alloc[MAX_THREADS];
struct itimerval alarm;

struct sigaction sched_signal;


struct Tque    ready_queue;
struct Task     *Current_Thread=NULL;

void schedule()
{
//cs printf("activated\n");
 if(Current_Thread==NULL) // Main Thread was executing previously..
 {
  TERMINATED=0;
  if(!isempty(&ready_queue))       // If there are more threads...
  {
   Current_Thread=top(&ready_queue);
//c   printf("yes!! %p\n", Current_Thread);
   Global_id=Current_Thread->thread_id;

   swapcontext(&Main, &(Current_Thread->threads_context));
  }
  else                // If all threads are dead...
  {
//c   printf("main is getting set!\n");
   setcontext(&Main);
  }
 }
 else     //if someother thread was executing...
 {
  struct Task* tmp=next(&ready_queue, Current_Thread);
//c  printf("other way around and %p and current:%p\n", tmp, Current_Thread);
  if(tmp==NULL)  //if this was the last thread in the queue....
  {     //execute main fn.
//c   printf("it was null\n");
   if(TERMINATED==1)
   {
//c    printf("its gone!!\n");
    TERMINATED=0;
    remov(&ready_queue, Global_id);
    Current_Thread=NULL;
    setcontext(&Main);
   }
   else
   {
    struct Task* tmp1=Current_Thread;
    Current_Thread=NULL;
    swapcontext(&(tmp1->threads_context), &Main); 
   }
   
  }
  else
  {
   struct Task* tmp2=Current_Thread;
   Current_Thread=tmp;
   if(TERMINATED==1)
   {
    TERMINATED=0;
    remov(&ready_queue, Global_id);
    Global_id=tmp->thread_id;
//c    printf("context set for %p\n", tmp);

    setcontext(&(tmp->threads_context));
   }
   else
   {
    Global_id=tmp->thread_id;
//c    printf("running:%p\n", tmp);
    swapcontext(&(tmp2->threads_context), &(tmp->threads_context)); 
   }
   
  }
 }
}

void Kill_Task(void *arg)
{
//c printf("its coming!!\n");
 int t_id=*((int*)arg);
//c printf("terminator for %d started\n", Global_id);
 TERMINATED=1;
 schedule();
}


void Init_Task()
{
 sched_signal.sa_handler=schedule;
 sigemptyset(&sched_signal.sa_mask);
 sched_signal.sa_flags=0;
 sigaction(SIGPROF, &sched_signal, NULL);
 ready_queue.count=0;
 int i=-1;
 while(++i<MAX_THREADS)
  alloc[i]=0;
 //for the first time alarm after 0 sec....
 //for every TIME_SLOT secs...
 alarm.it_value.tv_sec=0;
 alarm.it_value.tv_usec=TIME_SLOT;
 //next time alarm after every TIME_SLOT secs...
 alarm.it_interval.tv_sec=0;
 alarm.it_interval.tv_usec=TIME_SLOT;
//c printf("Initialized\n");
}

void Init_Semaphore(struct Task_Semaphore* mut, int value)
{
 mut->count=value;
 
}

void change_Semaphore(struct Task_Semaphore* mut, int value)
{
 sigset_t old, new;
 sigemptyset(&new);
 sigaddset(&new, SIGPROF);
 sigprocmask(SIG_BLOCK, &new, &old);
 if(mut->count > value)
 {
  mut->count-=value;
  sigprocmask(SIG_SETMASK, &old, NULL);
  return;
 }
 else
 {
  insert(&(mut->wait_queue), rem_and_poi(&ready_queue, Current_Thread));
  sigprocmask(SIG_SETMASK, &old,NULL);
  schedule();
 }
}
void create_task(struct Task* target, void (*fn)(void*), void * param)
{


  if(getcontext(&(target->threads_context))==-1)
  {
   printf("getcontext error!\n");
   exit(1);
  }
  
  (target->threads_context).uc_stack.ss_sp=malloc(STACK_SIZE);
  (target->threads_context).uc_stack.ss_size=STACK_SIZE;
  (target->threads_context).uc_stack.ss_flags=0;

  

  int id=-1;
  while(++id<10 && alloc[id]!=0);

  if(id==10)
  {
   printf("allocation error\n");
   exit(1);
  }
  target->thread_id=id;
  alloc[id]=1;
  //Delete[id]=malloc(sizeof(ucontext_t));
//c  printf("yyyy\n");
  if(getcontext(&Delete[id])==-1)
  {
   printf("delete context error!\n");
   exit(1);
  }
  
  Delete[id].uc_link=0;
  Delete[id].uc_stack.ss_sp=malloc(STACK_SIZE);
  Delete[id].uc_stack.ss_size=STACK_SIZE;
  Delete[id].uc_stack.ss_flags=0;
  (target->threads_context).uc_link=&Delete[id];
  makecontext(&Delete[id], (void*)&Kill_Task, 0);
  makecontext(&(target->threads_context), fn, 0);

  target->next=NULL;
  //if this is the first thread, then set the timer...
  if(CURRENT_THREAD_COUNT++==0)
  {
    if(setitimer(ITIMER_PROF, &alarm, NULL)==-1)
    {
     //some shit is happening with setitimer
     printf("Itimer error:%s", strerror(errno));
     exit(1);
    }
//c    printf("timer set!\n");
  }
  //thread should be added to queue...
  //so block all the SIGPROF signal until this is added....
  sigset_t new,old;
  sigemptyset(&new);
  sigaddset(&new, SIGPROF);
  sigprocmask(SIG_BLOCK, &new, &old);
  if(isempty(&ready_queue)!=0)
  {
//c   printf("first added to queue %p\n", target);
   insert(&ready_queue, target);
  }
  else
  {
//c   printf("next added to queue %p\n", target);
   insert(&ready_queue, target);
  }
  //sleep(5);
  //restore back the old mask...
  sigprocmask(SIG_SETMASK, &old, NULL);
}  


int Sigh=0;
void sample()
{
 printf("success!!\n");
 while(Sigh==0);
 printf("completed!!\n");
 //setcontext(&Main);
}

void samps()
{
 
 printf("success2\n");
 Sigh=1;
}
void sam3()
{
 printf("started3\n");
 //while(1);
 printf("success3\n");
}
int main(int argc, char *argv[])
{
 Init_Task();
 struct Task t,p,q;
 int a=5;
 create_task(&t, (void*)&sample, (void*)&a);
 create_task(&p, (void*)&samps, (void*)&a);
 create_task(&q, (void*)&sam3, (void*)&a);
 //getcontext(&Main);
 printf("yeasss\n");
 while(1);
 //exit(0);
}

The associated "thque.c" is as follows:


struct  Task
{
 int user_priority;
 ucontext_t threads_context;
 struct Task *next;
 int thread_id;
};
struct Tque
{
 struct Task *head;
 struct Task *tail;
 int count;
 
};

struct Task_Semaphore
{
 struct Tque wait_queue;
 int count;
};

void display(struct Tque *u)
{
 struct Task *tmp=u->head;
 while(tmp!=NULL)
 {
  printf("%p ", tmp);
  tmp=tmp->next;
 }
 printf("\n");
}
void insert(struct Tque *q, struct Task *t)
{
 if(q->count==0)
 {
  

  t->next=NULL;
  q->head=q->tail=t;
  
 }
 else
 {
  t->next=NULL;
  (q->tail)->next=t;
  q->tail=(q->tail)->next;
 }
 q->count++;
//c printf("insert:");
// display(q);
}

void remov(struct Tque *q, int id)
{
 struct Task *tmp=q->head, *prev;
 while(tmp!=NULL && tmp->thread_id!=id)
 {
  prev=tmp;
  tmp=tmp->next;
 }
 if(tmp!=NULL)
 {
//c  printf("getting deleted\n");
  if(tmp==q->head)
  {
//c   printf("first one!\n");
   q->head=(q->head)->next;
   //free(tmp);
  }
  else
  {
   prev->next=(prev->next)->next;
   //free(tmp);
  }
  q->count--;
 }
 else
 {
  printf("node with id=%d not found!\n",id);
 }
//c printf("delete:");
// display(q);
}

int isempty(struct Tque *q)
{
//c printf("count:%d\n",q->count );
 return q->count==0;
}

struct Task* top(struct Tque* q)
{
 if(q!=NULL)
 {
//c  printf("correct!\n");
  return q->head;
 }
 else
 {
  return NULL;
 }
}

struct Task* next(struct Tque* q, struct Task* cur)
{
 return cur->next;
}

struct Task* rem_and_poi(struct Tque* q, struct Task* to)
{
 struct Task *tmp=q->head, *tmp1=NULL;
 while(tmp!=to)
 {
  tmp1=tmp;
  tmp=tmp->next;
 }
 if(tmp1==NULL) //node is at head...
 {
  q->head=tmp->next;
  return tmp;
 }
 else   //node is in middle or may be end...
 {
  tmp1->next=tmp->next;
  return tmp;
 }
}

Note that in that thread library, the mutexes and sempahores weren't implemented fully.