!30 Attempt to fix SFTP Too many out-of-order packets.

From: @li_ning_jie 
Reviewed-by: @jiangheng12 
Signed-off-by: @jiangheng12

Making-remove_tags-function-more-efficient-from-quad.patch

@@ -1,73 +0,0 @@
-From e3a366a103ca699aaf57595c396ea205a50d2b16 Mon Sep 17 00:00:00 2001
-From: gilcu3 <828241+gilcu3@users.noreply.github.com>
-Date: Fri, 14 Sep 2018 10:19:24 -0400
-Subject: [PATCH 07/23] Making remove_tags function more efficient (from
- quadratic to linear time worst case)
-
----
- src/misc.cc | 47 +++++++++++++++++++++++++++++------------------
- 1 file changed, 29 insertions(+), 18 deletions(-)
-
-diff --git a/src/misc.cc b/src/misc.cc
-index db088c2..bc3fdb9 100644
---- a/src/misc.cc
-+++ b/src/misc.cc
-@@ -961,26 +961,37 @@ const xstring& shell_encode(const char *string,int len)
- void remove_tags(char *buf)
- {
-    int len=strlen(buf);
--   for(;;)
-+   int less = -1;
-+   for(int i = 0; i < len;i++)
-    {
--      char *less=strchr(buf,'<');
--      char *amp=strstr(buf,"&nbsp;");
--      if(!less && !amp)
--	 break;
--      if(amp && (!less || amp<less))
--      {
--	 amp[0]=' ';
--	 memmove(amp+1,amp+6,len-(amp+6-buf)+1);
--	 len-=amp+6-buf;
--	 buf=amp+1;
--	 continue;
-+      if(strcmp(buf + i, "&nbsp;") == 0){
-+        for(int j = 0; j < 6; j++)buf[i + j] = 0;
-+        buf[i] = ' ';
-+        i += 6;
-+        i--;
-+        continue;
-       }
--      char *more=strchr(less+1,'>');
--      if(!more)
--	 break;
--      memmove(less,more+1,len-(more+1-buf)+1);
--      len-=more+1-buf;
--      buf=less;
-+      if(buf[i] == '<'){
-+        less = i;
-+        continue;
-+
-+      }
-+      if(buf[i] == '>'){
-+        if(less != -1){
-+          for(int j = less; j <= i; j++)buf[j] = 0;
-+          less = -1;
-+        }
-+
-+      }
-+   }
-+   int zero = 0;
-+   for(int i = 0; i < len;i++)
-+   {
-+     while(zero < i && buf[zero] != 0)zero++;
-+     if(buf[i] != 0 and zero != i){
-+      buf[zero] = buf[i];
-+      buf[i] = 0;
-+     }
-    }
- }
- void rtrim(char *s)
--- 
-1.7.12.4
-

backport-Attempt-to-fix-SFTP-Too-many-out-of-order-.patch

@@ -0,0 +1,74 @@
+From fdb81537a2f854cf5e2b9dd95c7e5542bb5cd420 Mon Sep 17 00:00:00 2001
+From: Paul Dennis <padenn@github.com>
+Date: Mon, 29 Jul 2024 16:43:02 -0400
+Subject: [PATCH] Attempt to fix SFTP issue #636 Too many out-of-order packets.
+
+lftp supports making many SFTP DATA requests in parallel when downloading a file. Responses
+with data payloads from an sftp server can sometimes arrive in a different order from the
+original requests.
+
+The out of order data payload (packets) are added to the `ooo_chain` to be processed when the expected
+(but late) next requested packet arrives.
+
+A single packet being delayed while many parallel requests continue to be made can sometimes result in the
+`Too many out-of-order packets` message if the `ooo_chain` buffer of 64 entries is exceeded.
+
+This fix checks the remaining capacity of the `ooo_chain` and pending requests (`RespQueueSize()`)
+and will avoid sending new requests for more data until the `ooo_chain` is processed or `RespQueueSize()`
+shrinks. Adding this constraint does mean that `sftp:max-packets-in-flight` will be limited to
+the current `ooo_chain` length of 64.
+
+I was able to reliably reproduce the issue with a 4gb download and verify the fix.
+---
+ src/SFtp.cc | 8 ++++++--
+ src/SFtp.h  | 1 +
+ 2 files changed, 7 insertions(+), 2 deletions(-)
+
+diff --git a/src/SFtp.cc b/src/SFtp.cc
+index 6c818dfa..5879a631 100644
+--- a/src/SFtp.cc
++++ b/src/SFtp.cc
+@@ -344,6 +344,7 @@ void SFtp::Init()
+    size_write=0x8000;
+    use_full_path=false;
+    flush_timer.Set(0,500);
++   max_out_of_order=64;
+ }
+ 
+ SFtp::SFtp() : SSH_Access("SFTP:")
+@@ -892,7 +893,7 @@ void SFtp::HandleExpect(Expect *e)
+ 	 {
+ 	    LogNote(9,"put a packet with id=%d on out-of-order chain (need_pos=%lld packet_pos=%lld)",
+ 	       reply->GetID(),(long long)(pos+file_buf->Size()),(long long)r->pos);
+-	    if(ooo_chain.count()>=64)
++	    if(ooo_chain.count()>=max_out_of_order)
+ 	    {
+ 	       LogError(0,"Too many out-of-order packets");
+ 	       Disconnect();
+@@ -1192,7 +1193,10 @@ int SFtp::Read(Buffer *buf,int size)
+    {
+       // keep some packets in flight.
+       int limit=(entity_size>=0?max_packets_in_flight:max_packets_in_flight_slow_start);
+-      if(RespQueueSize()<limit && !file_buf->Eof())
++      int ooo_queue_available=max_out_of_order-ooo_chain.count();
++      int current_in_flight=RespQueueSize();
++      if(RespQueueSize()<limit && !file_buf->Eof()
++         && current_in_flight < ooo_queue_available)
+       {
+ 	 // but don't request much after possible EOF.
+ 	 if(entity_size<0 || request_pos<entity_size || RespQueueSize()<2)
+diff --git a/src/SFtp.h b/src/SFtp.h
+index 50888b37..2099440c 100644
+--- a/src/SFtp.h
++++ b/src/SFtp.h
+@@ -755,6 +755,7 @@ private:
+    int size_read;
+    int size_write;
+    bool use_full_path;
++   int max_out_of_order;
+ 
+ protected:
+    void SetError(int code,const Packet *reply);
+-- 
+2.33.0
+

fixed-sorting-of-FileSet-when-some-files-begin-with-.patch

@@ -1,56 +0,0 @@
-From b70425a3af79f387295acfa5246d6d0d8ba4e49c Mon Sep 17 00:00:00 2001
-From: "Alexander V. Lukyanov" <lavv17f@gmail.com>
-Date: Sat, 13 Oct 2018 00:10:27 +0300
-Subject: [PATCH 06/23] fixed sorting of FileSet when some files begin with a
- tilde (fixes #476)
-
-reason: FileSet::Merge no longer sorts the files.
----
- src/FileSet.h    | 1 +
- src/NetAccess.cc | 9 +++++----
- 2 files changed, 6 insertions(+), 4 deletions(-)
-
-diff --git a/src/FileSet.h b/src/FileSet.h
-index 88c9563..e686e1a 100644
---- a/src/FileSet.h
-+++ b/src/FileSet.h
-@@ -204,6 +204,7 @@ public:
-    void	 rewind() { ind=0; }
-    FileInfo *curr();
-    FileInfo *next();
-+   FileInfo *borrow_curr() { return Borrow(ind--); }
- 
-    void	 LocalRemove(const char *dir);
-    void	 LocalUtime(const char *dir,bool only_dirs=false,bool flat=false);
-diff --git a/src/NetAccess.cc b/src/NetAccess.cc
-index 62dd8ab..98a8329 100644
---- a/src/NetAccess.cc
-+++ b/src/NetAccess.cc
-@@ -541,19 +541,20 @@ do_again:
- got_fileset:
-       if(set)
-       {
--	 bool need_resort=false;
- 	 set->rewind();
- 	 for(file=set->curr(); file!=0; file=set->next())
- 	 {
- 	    // tilde is special.
- 	    if(file->name[0]=='~')
- 	    {
-+	       // can't just update the name - it will break sorting
-+	       file=set->borrow_curr();
- 	       file->name.set_substr(0,0,"./");
--	       need_resort=true;
-+	       if(!result)
-+		  result=new FileSet();
-+	       result->Add(file);
- 	    }
- 	 }
--	 if(need_resort && !result)
--	    result=new FileSet; // Merge will sort the names
- 	 if(result)
- 	 {
- 	    result->Merge(set);
--- 
-1.7.12.4
-

lftp.spec

@@ -1,16 +1,18 @@
 Summary:	A sophisticated file transfer program
 Name:		lftp
-Version:	4.8.4
-Release:	2
+Version:	4.9.1
+Release:	5
 License:	GPLv3+
 URL:		http://lftp.yar.ru/
 Source0:        http://lftp.yar.ru/ftp/%{name}-%{version}.tar.xz
 BuildRequires:	ncurses-devel, gnutls-devel, perl-generators, pkgconfig, gettext
 BuildRequires:	gettext readline-devel, zlib-devel, gcc-c++ desktop-file-utils
+BuildRequires:  chrpath
+Requires:       %{name}-help
 
-Patch1:         lftp-4.0.9-date_fmt.patch
-Patch6000:      fixed-sorting-of-FileSet-when-some-files-begin-with-.patch
-Patch6001:      Making-remove_tags-function-more-efficient-from-quad.patch
+Patch0:         lftp-4.0.9-date_fmt.patch
+Patch1:         quit-while-source-file-increased.patch
+Patch2:         backport-Attempt-to-fix-SFTP-Too-many-out-of-order-.patch
 
 %description
 LFTP is a sophisticated file transfer program supporting a number of
@@ -52,6 +54,11 @@ desktop-file-install    \
 --dir=%{buildroot}%{_datadir}/applications      \
 %{buildroot}/%{_datadir}/applications/lftp.desktop
 
+chrpath -d %{buildroot}/%{_libdir}/%{name}/%{version}/*.so*
+
+mkdir -p %{buildroot}/etc/ld.so.conf.d
+echo "%{_libdir}/lftp/%{version}" > %{buildroot}/etc/ld.so.conf.d/%{name}-%{_arch}.conf
+
 %ldconfig_scriptlets
 
 %files
@@ -65,6 +72,7 @@ desktop-file-install    \
 %{_datadir}/applications/*.*
 %{_datadir}/icons/hicolor/*/apps/*
 %exclude %{_libdir}/*.so
+%config(noreplace) /etc/ld.so.conf.d/*
 
 %files scripts
 %{_datadir}/lftp
@@ -78,5 +86,38 @@ desktop-file-install    \
 
 
 %changelog
+* Sun Oct 13 2024 liningjie <liningjie@xfusion.com> - 4.9.1-5
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC:Attempt to fix SFTP Too many out-of-order packets.
+
+* Fri Oct 29 2021 zengweifeng <zwfeng@huawei.com> - 4.9.1-4
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC:fix bug of quiting while source file increased
+
+* Mon Nov 09 2020 quanhongfei <quanhongfei@huawei.com> - 4.9.1-3
+- Type:requirement
+- ID:NA
+- SUG:NA
+- DESC:add lftp-help dependency for lftp
+
+* Tue Aug 18 2020 chenyaqiang <chenyaqiang@huawei.com> - 4.9.1-2
+- rebuild for package build
+
+* Thu Apr 14 2020 zhuchengliang <zhuchengliang4@huawei.com> - 4.9.1-1
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC: upgrade package
+
+* Fri Feb 28 2020 zhuchengliang <zhuchengliang4@huawei.com> - 4.8.4-3
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC: remove rpath and runpath of exec files and libraries
+
 * Wed Aug 28 2019 openEuler Buildteam <buildteam@openeuler.org> - 4.8.4-2
 - Package init

quit-while-source-file-increased.patch

@@ -0,0 +1,57 @@
+From 3cabbf03f80e78a69a0b9e797050a1dacf5c26da Mon Sep 17 00:00:00 2001
+From: sherlock2010 <15151851377@163.com>
+Date: Thu, 28 Oct 2021 18:36:53 +0800
+Subject: [PATCH] Quit while source file increased
+
+---
+ src/FileCopy.cc | 7 +++++++
+ src/FileCopy.h  | 1 +
+ 2 files changed, 8 insertions(+)
+
+diff --git a/src/FileCopy.cc b/src/FileCopy.cc
+index ead65a2..361958e 100644
+--- a/src/FileCopy.cc
++++ b/src/FileCopy.cc
+@@ -1137,6 +1137,7 @@ void FileCopyPeerFA::OpenSession()
+ 	 debug((10,"copy dst: seek past eof (seek_pos=%lld, size=%lld)\n",
+ 		  (long long)seek_pos,(long long)e_size));
+ 	 eof=true;
++	 fileincreased=true;
+ 	 if(date==NO_DATE || date==NO_DATE_YET)
+ 	    return;
+       }
+@@ -1284,6 +1285,11 @@ int FileCopyPeerFA::Put_LL(const char *buf,int len)
+    if(session->IsClosed())
+       OpenSession();
+ 
++   if(fileincreased)
++   {
++      SetError(_("file size increased during transfer"));
++      return -1;
++   }
+    off_t io_at=pos; // GetRealPos can alter pos, save it.
+    if(GetRealPos()!=io_at)
+       return 0;
+@@ -1352,6 +1358,7 @@ void FileCopyPeerFA::Init()
+ {
+    get_delay=0;
+    fxp=false;
++   fileincreased=false;
+    redirections=0;
+    can_seek=true;
+    can_seek0=true;
+diff --git a/src/FileCopy.h b/src/FileCopy.h
+index af97880..5a9e4cf 100644
+--- a/src/FileCopy.h
++++ b/src/FileCopy.h
+@@ -335,6 +335,7 @@ class FileCopyPeerFA : public FileCopyPeer
+    FileSet info;
+ 
+    bool fxp;   // FXP (ftp<=>ftp copy) active
++   bool fileincreased;
+ 
+    UploadState upload_state;
+    int redirections;
+-- 
+1.8.3.1
+